linsc 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 340aa6ad3472d2fe72211eaa0dba3a3bede9d01b
+  data.tar.gz: 6907b89561b075e9d82a15446cb0ca5ad88816c2
+SHA512:
+  metadata.gz: 5fc1790ddf980b7f7068e7bfd41bfa1fe70d183a4c5a5209f7782d8a8aec721843c08b8d2436866a59803d3f4f98ebbfcf7aafad6adb4b83c99d02ee6650d9b2
+  data.tar.gz: 9f41a2f305f1b0b170d7411d2fd81230c57869d0b43d6dd8598a4e23f48e9e903371e947dbd91ef128270ed676f520b81db37ce798049c86da9488b5c7b7c3da

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.3.0
+before_install: gem install bundler -v 1.11.2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in linsc.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Dan Molloy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,40 @@
+# Linsc
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/linsc`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'linsc'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install linsc
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/danmolloy/linsc.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "linsc"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/linsc

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'linsc'
+
+Linsc.new

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/data/agents.txt

@@ -0,0 +1,10 @@
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML like Gecko) Chrome/41.0.2227.1 Safari/537.36
+Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML like Gecko) Chrome/41.0.2228.0 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML like Gecko) Version/7.0.3 Safari/7046A194A
+Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1
+Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML like Gecko) Chrome/40.0.2214.93 Safari/537.36
+Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)
+Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)
+Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0
+Mozilla/5.0 (X11; OpenBSD amd64; rv:28.0) Gecko/20100101 Firefox/28.0

data/lib/linsc.rb

@@ -0,0 +1,159 @@
+require 'mechanize'
+require 'i18n'
+require 'fileutils'
+require 'csv'
+require 'optparse'
+require 'pathname'
+require_relative './linsc/merger'
+require_relative './linsc/cross_ref'
+require_relative './linsc/csv_handlers'
+require_relative './linsc/duck'
+require_relative './linsc/lin'
+
+
+class Linsc
+  include CSVHandlers
+
+  def merge
+    merge_map = {'First Name' => 'First Name', 'Last Name' => 'Last Name', 'E-mail Address' => 'Email',
+                  'Company' => 'Employer Organization Name 1', 'Job Title' => 'Employer 1 Title',
+                  'Recruiter' => 'LIN 1st Degree'}
+    Merger.new(@working_dir, @merge_path, merge_map).merge
+  end
+
+  def crossref
+     CrossRef.new(input_dir: @working_dir, child_path: @merge_path,
+     master_path: @sf_path, output_path: @crossref_path, options: @options)
+  end
+
+  def duck
+    DuckScraper.new(@working_dir, @crossref_path, @ddg_path, @options).find_profiles
+  end
+
+  def lin
+    LinScraper.new(@working_dir, @ddg_path, @options).start
+  end
+
+  def map_history_ids
+    puts "Mapping ids to history"
+    CrossRef.new(input_dir: @working_dir, child_path: @working_dir + "contact_employment_insert.csv",
+    master_path: @working_dir + "history_ref.csv", output_path: @working_dir + "contact_employment_insert_with_ids.csv",
+    options: {:noproxy => false, :update => true, :insert => false},
+    master_lookup_field: 'LIN ID', child_lookup_field: 'LIN ID',
+    master_secondary_lookups: nil, static_values: nil)
+    CrossRef.new(input_dir: @working_dir, child_path: @working_dir + "contact_education_insert.csv",
+    master_path: @working_dir + "history_ref.csv", output_path: @working_dir + "contact_education_insert_with_ids.csv",
+    options: {:noproxy => false, :update => true, :insert => false},
+    master_lookup_field: 'LIN ID', child_lookup_field: 'LIN ID',
+    master_secondary_lookups: nil, static_values: nil)
+    exit
+  end
+
+  def confirm_restart(first=true)
+    if first
+      puts "Are you sure you want to restart the project? This will delete all data except the original inputs.\n(y/n)"
+    else
+      puts "Unknown input. Please enter (y/n)"
+    end
+    input = gets.chomp
+    if input.downcase == 'y'
+      return true
+    elsif input.downcase == 'n'
+      return false
+    else
+      confirm_restart(false)
+    end
+  end
+  def restart_project
+    files = [@merge_path, @crossref_path, @ddg_path, @working_dir + "contact_update.csv",
+       @working_dir + "contact_insert.csv", @working_dir + "contact_employment_update.csv",
+        @working_dir + "contact_employment_insert.csv", @working_dir + "contact_education_update.csv",
+         @working_dir + "contact_education_insert.csv"]
+    files.each do |f|
+      File.delete(f) if File.exist?(f)
+    end
+  end
+
+  def initialize
+    @options = {:noproxy => false, :update => false, :insert => false}
+    @working_dir = Pathname.pwd
+    @merge_path = @working_dir + 'merged.csv'
+    @sf_path = @working_dir + 'sf_ref.csv'
+    @crossref_path = @working_dir + 'crossref.csv'
+    @ddg_path = @working_dir + 'ddg.csv'
+
+    parser = OptionParser.new do|opts|
+      opts.banner = "Must specify update or insert (or both)"
+      opts.on('-u', '--update', 'Tell scraper to fetch fresh data for existing Salesforce records') do
+        @options[:update] = true;
+      end
+
+      opts.on('-i', '--insert', 'Tell scraper to fetch data for new connections not yet in Salesforce') do
+        @options[:insert] = true;
+      end
+
+      opts.on('-n', '--noproxy', 'Do not use any proxies') do
+        @options[:noproxy] = true;
+      end
+
+      opts.on('-e', '--history', 'Map Contact IDs to education/employment histories for new connections') do
+        map_history_ids
+      end
+
+      opts.on('-r', '--restart', 'Restart the project from beginning with the same inputs. WARNING: This will delete all scraped data.') do
+        if confirm_restart(true)
+          restart_project
+          puts "project files deleted"
+        else
+          puts "exiting"
+          exit
+        end
+      end
+
+      opts.on('-h', '--help', 'Displays Help') do
+        puts opts
+        exit
+      end
+    end.parse!
+
+    required_sf_fields = ['LIN ID', 'Email', 'Contact ID']
+    sf_headers = get_headers(@sf_path)
+    required_sf_fields.each do |field|
+      unless sf_headers.include?(field)
+        puts "The SF reference sheet must include the #{field} field."
+        exit
+      end
+    end
+
+    if File.exist?(@ddg_path)
+      ids = []
+      CSV.foreach(@crossref_path, headers: true) do |row|
+        ids << row['Contact ID']
+      end
+      if ids.include?(nil) || ids.include?("")
+        @options[:insert] = true
+      else
+        @options[:insert] = false
+      end
+      if ids.any?{|id| id && id&.length > 0}
+        @options[:update] = true
+      else
+        @options[:update] = false
+      end
+      puts "\nResuming previous scraping. insert: #{@options[:insert]}, update: #{@options[:update]}, using proxies? #{!@options[:noproxy]}"
+    else
+      unless @options[:update] || @options[:insert]
+        puts "Must specify insert or update. See help for details with -h"
+        exit
+      end
+      puts "\nStarting new project. insert: #{@options[:insert]}, update: #{@options[:update]}, using proxies? #{!@options[:noproxy]}"
+    end
+
+    merge unless File.exist?(@ddg_path)
+    crossref unless File.exist?(@ddg_path)
+    duck
+    lin
+
+  end
+
+end

data/lib/linsc/cross_ref.rb

@@ -0,0 +1,113 @@
+require_relative 'csv_handlers'
+require 'securerandom'
+
+class CrossRef
+  include CSVHandlers
+  include SecureRandom
+
+  def initialize(input_dir:, child_path:, master_path:, output_path:,
+                 master_lookup_field: 'Email', child_lookup_field: 'Email',
+                 master_secondary_lookups: ['Email 2', 'Email 3'],
+                 static_values: {'Account Name' => 'Candidates'}, options:)
+    @input_dir, @child_path, @master_path, @output_path, @options =
+      input_dir, child_path, master_path, output_path, options
+    @master_lookup_field, @child_lookup_field, @master_secondary_lookups, @static_values =
+      master_lookup_field, child_lookup_field, master_secondary_lookups, static_values
+    @headers = get_headers(@master_path)
+    child_lookup_field == 'Email' ? @email_key = true : @email_key = false
+    @child_headers = get_headers(@child_path)
+    @child_headers.each do |child_header|
+      unless @headers.include?(child_header)
+        @headers << child_header
+      end
+    end
+    @static_values.each do |static_key, static_value|
+      unless @headers.include?(static_key)
+        @headers << static_key
+      end
+    end if @static_values
+    @child_length = %x(wc -l "#{@child_path}").split[0].to_i - 1
+    if File.exist?(@output_path)
+      File.delete(@output_path)
+    end
+    create_file(@output_path)
+    cross_ref
+  end
+
+  def cross_ref
+    master_data = CSV.read(@master_path, headers: true)
+    puts "sorting lookup values"
+    master_data = master_data.sort do |x, y|
+      a = x[@master_lookup_field]
+      b = y[@master_lookup_field]
+      a && b ? a <=> b : a ? -1 : 1
+    end
+    master_lookup_values = master_data.collect {|row| row[@master_lookup_field]&.downcase}
+    i = 0
+    CSV.foreach(@child_path, headers: true, encoding: 'utf-8') do |child_row|
+      i += 1
+      puts "email lookup - row: #{i}/#{@child_length}"
+      child_lookup_value = child_row[@child_lookup_field]&.downcase
+      if child_lookup_value&.include?('@') || !@email_key ## generalize this
+        match_index = master_lookup_values.bsearch_index do |master_lookup_value|
+           child_lookup_value && master_lookup_value ?
+                child_lookup_value <=> master_lookup_value : child_lookup_value ? -1 : 1
+        end
+        if !match_index
+          match_index = master_data.find_index do |master_row|
+            master_secondary_lookups = @master_secondary_lookups.collect{|x| x&.downcase}
+            master_secondary_lookups.include?(child_lookup_value)
+          end
+        end
+        if match_index
+          if @options[:update]
+            append_to_csv(@output_path, splice_rows(master_data[match_index], child_row))
+          end
+        else
+          if @options[:insert]
+            append_to_csv(@output_path, convert_row(child_row))
+          end
+        end
+      else
+        puts "missing lookup value"
+      end
+    end
+  end
+
+  def splice_rows(master_row, child_row)
+    unless master_row['LIN ID'] && master_row['LIN ID'].strip.length > 20
+      master_row['LIN ID'] = SecureRandom.hex(16)
+    end
+
+    child_row.each do |child_key, child_value|
+      if child_value && child_value.strip.length > 0
+        if master_row.has_key?(child_key)
+          master_row[child_key] = child_value
+        else
+          master_row << [child_key, child_value]
+        end
+      else
+        unless master_row.has_key?(child_key)
+          master_row << [child_key, child_value]
+        end
+      end
+    end
+    master_row_new = CSV::Row.new(@headers, [])
+    master_row.each do |key, value|
+      master_row_new[key] = value&.encode('utf-8', invalid: :replace, undef: :replace, replace: '#')
+    end
+    master_row_new
+  end
+
+  def convert_row(child_row)
+    master_row = CSV::Row.new(@headers, [])
+    master_row['LIN ID'] = SecureRandom.hex(16)
+    child_row.each do |child_key, child_value|
+      master_row[child_key] = child_value if master_row.has_key?(child_key)
+    end
+    @static_values.each do |static_key, static_value|
+      master_row[static_key] = static_value if master_row.has_key?(static_key)
+    end
+    master_row
+  end
+end

data/lib/linsc/csv_handlers.rb

@@ -0,0 +1,53 @@
+module CSVHandlers
+  def create_row(row, headers, encoding = nil)
+    values = []
+    headers.each do |header|
+      if encoding
+        values << row[header]&.encode(encoding)
+      else
+        values << row[header]
+      end
+    end
+    CSV::Row.new(headers, values)
+  end
+
+  def append_to_csv(file, row)
+    tries = 3
+    begin
+      f = CSV.open(file, "a+", headers: row.headers, force_quotes: true)
+      f << row
+      f.close
+    rescue
+      tries -= 1
+      if tries > 0
+        retry
+      else
+        puts "Unable to write to file #{file}"
+        puts "Make sure the file exists and is not open in any other programs and try again. If that does not work try restarting your computer, or restarting the project with the -r flag."
+        exit
+      end
+    end
+  end
+
+  def create_file(f)
+    unless File.exist?(f)
+      FileUtils.touch(f)
+      csv = CSV.open(f, "w+")
+      csv << @headers.collect {|x| x&.encode('utf-8')}
+      csv.close
+    end
+  end
+
+  def create_file_with_headers(f, headers)
+    unless File.exist?(f)
+      FileUtils.touch(f)
+      csv = CSV.open(f, "w+")
+      csv << headers.collect {|x| x&.encode('utf-8')}
+      csv.close
+    end
+  end
+
+  def get_headers(file)
+    CSV.open(file, headers: true, return_headers: true).shift.headers
+  end
+end