linsc 0.0.11 → 0.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d7427dce47840c832e198f854d4ab28eebc2be38
-  data.tar.gz: c441b1fab2b2fd62076999474d4ae299686591da
+  metadata.gz: a921c0d961c96ce6b775700e1937513f1e1b7c92
+  data.tar.gz: e849dd28fdaf09b7e5592cde52a27ee04045205d
 SHA512:
-  metadata.gz: 4937073b37327a5e8d77c1401056f47691d4310ebb31291c095884c5b89eacaa83789b968e1f1185f8899cc7371f63309c46680b93c7f2286899b8e75f4b574c
-  data.tar.gz: acb76982a619630663b5c048233d02b5053ef7e520f249dfefc749c3a2989614a46bd112b52da64886e4cf87f71dbde92128e335242be86bd5f7678714b88d58
+  metadata.gz: 20d14e5800c05b0417b1917e6983c44b05bf2b898d8e3cb601cd952ec846ff13489a2a6eb6368452bc0cb6681361688ea0504a979dc33bc0f253d7d23c6da0a5
+  data.tar.gz: f763c237238b6f116d69949f04dc9fdae4be1d716dcb2e936ba1f71b6519ea2f4d52f5e36b20bca0ff93f77f8e376c1e8d54f691e43fb01f4b0a5acc25d4ec80

data/README.md

@@ -1,34 +1,67 @@
 # Linsc
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/linsc`. To experiment with that code, run `bin/console` for an interactive prompt.
+## Installation
 
-TODO: Delete this and the text above, and describe your gem
+### Windows
 
-## Installation
+- Make sure Powershell is installed
+
+- Download the Windows Ruby installer from here: http://rubyinstaller.org/downloads/
+
+- Install ruby
 
-Add this line to your application's Gemfile:
+- Open Powershell and run the command:
+```ruby
+gem install linsc
+```
 
+- When installation is completed find your gem directory with:
 ```ruby
-gem 'linsc'
+gem environment
 ```
 
-And then execute:
+- Navigate to the path listed for INSTALLATION DIRECTORY, then gems -> linsc-(version) -> data
 
-    $ bundle
+- Here you will find an empty proxies.txt and recruiters.txt
 
-Or install it yourself as:
+- Fill the recruiters file with your recruiters, ordered by priority. One recruiter per line, comma separated like this:
+```
+LIN1,
+LIN2,
+LIN3,
+LIN4
+```
 
-    $ gem install linsc
+- Fill proxies.txt, one proxy per line in the form ip:port or ip:port:username:password if using a username/password with the proxies.
+```
+108.123.523.23:8080:admin:pass
+108.123.523.123:8080:admin:pass
+108.123.523.535:8080:admin:pass
+```
+(username and password optional)
 
 ## Usage
 
-TODO: Write usage instructions here
+- Download your linkedin connections export csv files and put them in an empty directory. Each of these files should have exactly the same filename as one of the lines from your recruiters.txt file.
+It is expected that these csvs will have the following fields: First Name, Last Name, E-mail Address, Company, Job Title. All other fields are ignored.
+
+- Run a Salesforce report and put the csv in the same directory as your LIN files. The required fields for the Salesforce file are: Contact ID, LIN ID, Email.
+This file should be named sf_ref.csv
+
+- In Powershell (or any terminal), navigate to the directory and run this command:
+```
+linsc
+```
 
-## Development
+- There are several optional flags you can provide to modify behaviour. To see the full list of available flags, run:
+```
+linsc -h
+```
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+- If all proxies get blocked the program will exit. Wait a few hours or a day for the proxies to cool off, then run linsc again. It will pick up where it left off.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+- When scraping is complete, import your data. If you had the -i flag set, you must import the contact_insert csv, then export a new report with the fields: LIN ID, Contact ID.
+Name this csv history_ref.csv and put it in the same folder as your data. Run linsc again with the -e flag set and the new Contact IDs will be mapped to the education and employment histories for the new contacts. You can now import these new history objects.
 
 ## Contributing
 

data/lib/linsc/lin.rb

@@ -167,6 +167,7 @@ class LinScraper
       sleep(@cooldown) if @noproxy
       page = agent.get(url)
       puts 'ACCESS GRANTED'
+      proxy.good if proxy
 
       return false unless page.at_css("#name") && page.css("#experience .positions .position")
       return false unless name_check(page.at_css("#name").text, "#{row['First Name']} #{row['Last Name']}")
@@ -184,7 +185,6 @@ class LinScraper
           end
         end
       end
-      proxy.good if proxy
       if match
         return [url, page]
       else

data/lib/linsc/merger.rb

@@ -28,7 +28,7 @@ class Merger
     emails = {}
     @lin_files.each do |pn|
       lin_file = pn.to_s
-      recruiter_name = lin_file.match(/LIN[^.]+/)[0]
+      recruiter_name = pn.basename.to_s.match(/LIN[^.]+/)[0]
       puts "merging #{recruiter_name}"
       clean_file = File.read(lin_file, encoding: 'windows-1252').strip
       CSV.parse(clean_file, headers: true, encoding: 'windows-1252') do |row|

data/lib/linsc/proxy.rb

@@ -1,19 +1,21 @@
 class Proxy
-  attr_accessor :ip, :port, :username, :password, :status, :last_used, :user_agent
+  attr_accessor :ip, :port, :username, :password, :status, :last_used, :user_agent, :burnout_time, :pages_before_burnout
 
-  def initialize(ip:, port: 80, username: nil, password: nil, status: nil, last_used: nil, user_agent: nil)
-    @ip, @port, @username, @password, @status, @last_used =
-                  ip, port, username, password, status, last_used
+  def initialize(ip:, port: 80, username: nil, password: nil, status: nil, last_used: nil, user_agent: nil, burnout_time: nil, pages_before_burnout: 0)
+    @ip, @port, @username, @password, @status, @last_used, @user_agent, @burnout_time, @pages_before_burnout =
+                  ip, port, username, password, status, last_used, user_agent, burnout_time, pages_before_burnout
   end
 
   def dead
     @status = 'dead'
     @last_used = Time.now
+    @burnout_time = Time.now
   end
 
   def good
     @status = 'good'
     @last_used = Time.now
+    @pages_before_burnout += 1
   end
 
   def good?

data/lib/linsc/proxy_handler.rb

@@ -35,6 +35,9 @@ class ProxyHandler
       best_proxy
     else
       puts "All proxies are dead. Wait a few hours before resuming."
+      @proxies.each do |proxy|
+        puts "ip: #{proxy.ip} .... pages before burnout: #{proxy.pages_before_burnout} .... time of burnout: #{proxy.burnout_time}"
+      end
       exit
     end
   end

data/linsc.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "linsc"
-  spec.version       = "0.0.11"
+  spec.version       = "0.0.12"
   spec.authors       = ["Dan Molloy"]
   spec.email         = ["danieljmolloy1@gmail.com"]
   spec.date = '2016-03-31'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: linsc
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.0.12
 platform: ruby
 authors:
 - Dan Molloy
@@ -112,8 +112,6 @@ files:
 - lib/linsc/parsers.rb
 - lib/linsc/proxy.rb
 - lib/linsc/proxy_handler.rb
-- linsc-0.0.1.gem
-- linsc-0.0.2.gem
 - linsc.gemspec
 homepage: https://github.com/danmolloy/linsc
 licenses: