appending 0.2 → 1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 511aa4fb2b48eab5db9cb290c7e58449d4d04e5ec88057d025ed48551a2c2d4c
-  data.tar.gz: f95e050a801ab8727f96500743ebd03c67cb745d93d6d93fda71939971655589
+  metadata.gz: 67bdb77eef8558d6e35c4b4459f6cb9fec576cd1b0fb139b521b42c5314d8471
+  data.tar.gz: ba9f19e7c87466eabf297bc3e1f685bf13ada1ad18cfd2d346a2b22e7edfb181
 SHA512:
-  metadata.gz: 62418ca562f10dd0dd2e69546c33639625f166d8f627e31e9b4104de76d32d12a16743824aa54ee01807c6bfb0d9b8e27f94162ee6cc2e0e7733c04c5871eff4
-  data.tar.gz: 507d4bcd5d681e0021db947dcd7e7011a2e3c36e25d0dc0d1266a1a76f2bc4a9de09aa91ac7daf15b7d7abb8539a6ae36c44271a6e14541a3199ab96ac0c7cbe
+  metadata.gz: 92635a6cb6d3117e32c58b5cc372dba8f5d161655cb7d60af3f3ed3200b2bd5f948e6586238a211f62ba5dae7d26e7f06c6a9182d73c0b5a0f000c710f554933
+  data.tar.gz: 2cf1e90ca32603cdb498107016b887b251f5118522641cb6f9ef1c8fec811a0033650acd0a4511164d4672cd80d03fce317005663ed20a9f1c6f701c059a94a3

data/lib/appending.rb

@@ -0,0 +1,455 @@
+require 'csv'
+require 'email_verifier'
+require 'csv-indexer'
+require 'simple_cloud_logging'
+
+module BlackStack
+    module Appending
+        @@logger = nil
+        @@report = nil
+        @@indexes = []
+        @@verifier_url = 'https://connectionsphere.com/api1.0/emails/verify.json'
+        @@verifier_api_key = nil
+        @@email_fields = []
+        @@phone_fields = []
+        @@company_domain_fields = []
+
+        ## @@logger
+        def self.set_logger(logger)
+            @@logger = logger
+        end
+
+        def self.logger
+            @@logger
+        end
+
+        ## @@indexes
+        def self.add_index(index)
+            expected = [:company_name, :first_name, :last_name]
+
+            # validation: keys must be `[:company_name, :first_name, :last_name]`
+            if !index.keys.eql?(expected)
+                raise "Invalid index: #{index.keys}. Expected: #{expected}."
+            end
+            # add the index
+            @@indexes << index
+        end
+
+        def self.set_indexes(indexes)
+            @@indexes = indexes
+        end
+        
+        def self.indexes
+            @@indexes
+        end
+
+        # @@report
+        def self.report
+            @@report
+        end
+
+        # @@verifier_url
+        def self.set_verifier_url(url)
+            @@verifier_url = url
+        end
+        
+        def self.verifier_url
+            @@verifier_url
+        end
+
+        # @@verifier_api_key
+        def self.set_verifier_api_key(key)
+            @@verifier_api_key = key
+        end
+
+        def self.verifier_api_key
+            @@verifier_api_key
+        end
+
+        # @@email_fields
+        def self.set_email_fields(fields)
+            @@email_fields = fields
+        end
+
+        def self.email_fields
+            @@email_fields
+        end
+
+        # @@phone_fields
+        def self.set_phone_fields(fields)
+            @@phone_fields = fields
+        end
+
+        def self.phone_fields
+            @@phone_fields
+        end
+
+        # @@company_domain_fields
+        def self.set_company_fields(fields)
+            @@company_domain_fields = fields
+        end
+
+        def self.company_domain_fields
+            @@company_domain_fields
+        end
+
+        # set configuration
+        def self.set(h)
+            errors = []
+
+            # validation: if :indexes is present, it must be an array of objects BlackStack::CSVIndexer::Index
+            if h[:indexes]
+                if !h[:indexes].is_a?(Array)
+                    errors << "Invalid :indexes: #{h[:indexes].class}. Expected: Array."
+                else
+                    h[:indexes].each { |index|
+                        if !index.is_a?(BlackStack::CSVIndexer::Index)
+                            errors << "Invalid :indexes: #{index.class}. Expected: BlackStack::CSVIndexer::Index."
+                        end
+                    }
+                end
+            end
+
+            # validation: if :verifier_url is present, it must be a string
+            errors << ":verifier_url must be a string." if h[:verifier_url] && !h[:verifier_url].is_a?(String)
+
+            # validation: if :verifier_api_key is present, it must be a string
+            errors << ":verifier_api_key must be a string." if h[:verifier_api_key] && !h[:verifier_api_key].is_a?(String)
+
+            # validation: if :email_fields is present, it must be an array of strings
+            if h[:email_fields]
+                if !h[:email_fields].is_a?(Array)
+                    errors << "Invalid :email_fields: #{h[:email_fields].class}. Expected: Array."
+                else
+                    h[:email_fields].each { |field|
+                        if !field.is_a?(String)
+                            errors << "Invalid :email_fields: #{field.class}. Expected: String."
+                        end
+                    }
+                end
+            end
+
+            # validation: if :phone_fields is present, it must be an array of strings
+            if h[:phone_fields]
+                if !h[:phone_fields].is_a?(Array)
+                    errors << "Invalid :phone_fields: #{h[:phone_fields].class}. Expected: Array."
+                else
+                    h[:phone_fields].each { |field|
+                        if !field.is_a?(String)
+                            errors << "Invalid :phone_fields: #{field.class}. Expected: String."
+                        end
+                    }
+                end
+            end
+
+            # validation: if :company_domain_fields is present, it must be an array of strings
+            if h[:company_domain_fields]
+                if !h[:company_domain_fields].is_a?(Array)
+                    errors << "Invalid :company_domain_fields: #{h[:company_domain_fields].class}. Expected: Array."
+                else
+                    h[:company_domain_fields].each { |field|
+                        if !field.is_a?(String)
+                            errors << "Invalid :company_domain_fields: #{field.class}. Expected: String."
+                        end
+                    }
+                end
+            end
+
+            # mapping
+            @@indexes = h[:indexes] if h[:indexes]
+            @@verifier_url = h[:verifier_url] if h[:verifier_url]
+            @@verifier_api_key = h[:verifier_api_key] if h[:verifier_api_key]
+            @@email_fields = h[:email_fields] if h[:email_fields]
+            @@phone_fields = h[:phone_fields] if h[:phone_fields]
+            @@company_domain_fields = h[:company_domain_fields] if h[:company_domain_fields]
+        end
+
+        # return true if the domain get any random address as valid
+        #
+        # This is a support method for the `append` methods.
+        # The end-user should not call this method directly.
+        # 
+        def self.catch_all?(domain)
+            BlackStack::Appending.verify("008e77980535470e848a4ca859a83db0@#{domain}")
+        end
+
+        # verify an email address using the AWS IP address of our website, wich is more reliable
+        #
+        # This is a support method for the `append` methods.
+        # The end-user should not call this method directly.
+        # 
+        def self.verify(email)
+            url = @@verifier_url
+            params = {
+                :email => email,
+            }
+            res = BlackStack::Netting::call_get(url, params)
+            parsed = JSON.parse(res.body)
+            parsed['status'] == 'success'
+        end
+
+        # verify an email address
+        def self.append(fname, lname, domain)
+            ret = []
+            if !catch_all?(domain)
+                EmailVerifier.config do |config|
+                    config.verifier_email = "leandro.sardi@expandedventure.com"
+                end
+                [
+                    "#{fname}@#{domain}",
+                    "#{lname}@#{domain}",
+
+                    "#{fname}.#{lname}@#{domain}",
+                    "#{lname}.#{fname}@#{domain}",
+
+                    "#{fname}#{lname}@#{domain}",
+                    "#{lname}#{fname}@#{domain}",
+
+                    "#{fname[0]}#{lname}@#{domain}",
+                    "#{fname[0]}.#{lname}@#{domain}",
+                ].each { |email|
+                    ret << email.downcase if verify(email)
+                }
+            end
+            ret
+        end
+
+        # This is a support method for the `append` methods.
+        # The end-user should not call this method directly.
+        def self.cleanup_fname(name)
+            return '' if name.nil?
+            a = name.split(/[^a-zA-Z]/)
+            a.size > 0 ? a[0] : ''
+        end
+
+        # This is a support method for the `append` methods.
+        # The end-user should not call this method directly.
+        def self.cleanup_lname(name)
+            return '' if name.nil?
+            a = name.split(/[^a-zA-Z]/)
+            a.size > 1 ? a[1] : ''
+        end
+
+        # This is a support method for the `append` methods.
+        # The end-user should not call this method directly.
+        def self.cleanup_company(company)
+            return '' if company.nil?
+            ret = ''
+            # stage 1: remove company-type suffixes
+            company = company.split(/ at /).last
+            company.gsub!(/LLC/, '')
+            company.gsub!(/Inc/, '')
+            company.strip! # remove leading and trailing spaces
+            # stage 2: remove LinkedIn suffixes            
+            company.gsub!(/\(\d\d\d\d - Present\)/, '')
+            company.strip! # remove leading and trailing spaces
+            # stege 3: remove non-alphanumeric characters
+            company.gsub!(/\.$/, '')  
+            company.gsub!(/\,$/, '') 
+            # stege 4: remove extra spaces
+            company.gsub!(/[^a-zA-Z0-9,\.\-\s]/, '') # remove extra spaces
+            company.strip! # remove leading and trailing spaces
+            # stage 5: choose the first part of the company name
+            company.split(' ').each { |word|
+                ret += word + ' '
+                #break if word.size >= 5 || ret.split(' ').size > 2
+                break if ret.split(' ').size > 2
+            } 
+            ret.strip!
+            # return
+            ret
+        end
+
+        # Find a person in the indexes by its full name and company name.
+        # Append all the information in the index row.
+        def self.find_persons_with_full_name(name, cname)
+            l = BlackStack::Appending.logger || BlackStack::DummyLogger.new
+
+            l.logs "Guessing fname from #{name}... "
+            fname = BlackStack::Appending::cleanup_fname(name)
+            l.logf fname
+
+            l.logs "Guessing lname from #{name}... "
+            lname = BlackStack::Appending::cleanup_lname(name)
+            l.logf lname
+
+            BlackStack::Appending.find_persons(fname, lname, cname)
+        end
+
+        # Find a person in the indexes by its first name, last name and company name.
+        # Append all the information in the index row.
+        def self.find_persons(fname, lname, cname)
+            l = BlackStack::Appending.logger || BlackStack::DummyLogger.new
+            h = {
+                :matches => [],
+                :enlapsed_seconds => 0,
+                :files_processed => 0,
+            }
+            # cleaning up company name
+            l.logs "Cleaning up company name #{cname}... "
+            cname = BlackStack::Appending::cleanup_company(cname)
+            l.logf cname
+            # looking for a record that matches with first name, last name and company name
+            appends = []
+            enlapsed_seconds = 0
+            files_processed = 0
+            BlackStack::Appending.indexes.each { |i|
+                l.logs "Searching into #{i.name}... "
+                ret = i.find([cname, fname, lname], false, nil)
+                # add the name of the index in the last position of the match
+                ret[:matches].each { |m| m.unshift(i.name.to_s) }
+                # add matches to the list
+                h[:matches] += ret[:matches]
+                # sum the total files and the total enlapsed seconds                
+                h[:enlapsed_seconds] += ret[:enlapsed_seconds]
+                h[:files_processed] += ret[:files_processed]
+                l.done
+            }
+            # update report
+            @@report = h
+            # return results
+            h[:matches].map { |m| BlackStack::Appending::Result.new(m) }
+        end
+
+        # Find a company in the indexes by its first name, last name and company name.
+        # Append all the information in the index row.
+        def self.find_persons_by_company(cname)
+            l = BlackStack::Appending.logger || BlackStack::DummyLogger.new
+            h = {
+                :matches => [],
+                :enlapsed_seconds => 0,
+                :files_processed => 0,
+            }
+            # looking for a record that matches with first name, last name and company name
+            appends = []
+            enlapsed_seconds = 0
+            files_processed = 0
+            BlackStack::Appending.indexes.each { |i|
+                l.logs "Searching into #{i.name}... "
+                ret = i.find([cname], true, nil)
+                # add the name of the index in the last position of the match
+                ret[:matches].each { |m| m.unshift(i.name.to_s) }
+                # add matches to the list
+                h[:matches] += ret[:matches]
+                # sum the total files and the total enlapsed seconds                
+                h[:enlapsed_seconds] += ret[:enlapsed_seconds]
+                h[:files_processed] += ret[:files_processed]
+                l.done
+            }
+            # update report
+            @@report = h
+            # return results
+            h[:matches].map { |m| BlackStack::Appending::Result.new(m) }
+        end
+
+        def self.find_verified_emails(fname, lname, cname)
+            l = BlackStack::Appending.logger || BlackStack::DummyLogger.new
+            emails = []
+            domains = []
+            verified_emails = []
+            # get lead emails from in the indexes
+            l.logs ("Searching index emails... ")
+                emails = BlackStack::Appending.find_persons(fname, lname, cname).map { |res| 
+                    res.emails 
+                }.flatten.uniq.reject { |email|
+                    email.to_s.empty?
+                }
+            l.done
+            # get company domains from the indexes
+            l.logs ("Searching index domains... ")
+                domains = BlackStack::Appending.find_persons_by_company(cname).map { |res| 
+                    res.company_domains
+                }.flatten.reject { |email|
+                    email.to_s.empty?
+                }.map { |domain|
+                    # normalize domain
+                    domain.to_s.gsub('www.', '').downcase
+                }.uniq
+            l.done
+            # verify all the emails found in the indexes
+            l.logs ("Verifying index emails... ")
+            emails.each { |email|
+                l.logs "Verifying #{email}... "
+                domain = email.split('@').last
+                verified_emails << email if BlackStack::Appending.verify(email) && !BlackStack::Appending.catch_all?(domain)
+                l.done
+            }
+            l.done
+            # appending with domains found in the indexes
+            l.logs ("Appending with domains... ")
+            domains.each { |domain|
+                l.logs "Appending with #{domain}... "
+                verified_emails += BlackStack::Appending.append(fname, lname, domain)
+                l.done
+            }
+            l.done
+            # return 
+            verified_emails.uniq
+        end
+
+        # 
+        class Result
+            # array of values.
+            # first 3 values are index name, key and row-number.
+            attr_accessor :match
+            
+            def initialize(a)
+                self.match = a
+            end
+
+            # From a given match (with the name of its index in the first position), get the value of a field by its name.
+            def value(field)
+                # get the index_name
+                index_name = match[0]
+                # get the index descriptor
+                index = BlackStack::CSVIndexer.indexes.select { |i| i.name == index_name }.first
+                # get position of the field into the hash descriptior
+                k = index.mapping.to_a.map { |m| m[0].to_s }.index(field.to_s)
+                # return nil if the field is not found
+                return nil if k.nil?
+                # get the field value
+                match[k+3].to_s
+            end
+
+            # Call value() method.
+            def val(field)
+                self.value(field)
+            end
+
+            # From a given match (with the name of its index in the first position), get the email addresses.
+            def emails()
+                keys = BlackStack::Appending.email_fields
+                ret = []
+                keys.each { |k|
+                    v = self.value(k)
+                    ret << v if v
+                }
+                ret
+            end
+
+            # From a given match (with the name of its index in the first position), get the phone numbers.
+            def phones()
+                keys = BlackStack::Appending.phone_fields
+                ret = []
+                keys.each { |k|
+                    v = self.value(k)
+                    ret << v if v
+                }
+                ret
+            end
+
+            # From a given match (with the name of its index in the first position), get the company domains.
+            def company_domains()
+                keys = BlackStack::Appending.company_domain_fields
+                ret = []
+                keys.each { |k|
+                    v = self.value(k)
+                    ret << v if v
+                }
+                ret
+            end
+
+        end # class Result
+
+    end # Appending
+end # BlackStack

metadata

@@ -1,15 +1,55 @@
 --- !ruby/object:Gem::Specification
 name: appending
 version: !ruby/object:Gem::Version
-  version: '0.2'
+  version: '1.0'
 platform: ruby
 authors:
 - Leandro Daniel Sardi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-12-19 00:00:00.000000000 Z
+date: 2022-12-20 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.2
+- !ruby/object:Gem::Dependency
+  name: email_verifier
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 - !ruby/object:Gem::Dependency
   name: blackstack-core
   requirement: !ruby/object:Gem::Requirement
@@ -136,8 +176,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- appending.rb
-homepage: https://rubygems.org/gems/pampa
+- lib/appending.rb
+homepage: https://rubygems.org/gems/appending
 licenses:
 - MIT
 metadata: {}

data/appending.rb

@@ -1,141 +0,0 @@
-require 'csv'
-require 'email_verifier'
-require 'nokogiri'
-
-puts "Appending is in development. Please check back later."
-
-=begin
-module BlackStack
-    module Appending
-        # This class is used to parse the HTML files downloaded from Sales Navigator and other sources.
-        module Parser
-            # parse search results pages from sales navigator, and save the company name and full name into a CSV file
-            def self.parse_sales_navigator_result_pages(search_name, l=nil)
-                # create logger if not passed
-                l = BlackStack::DummyLogger.new(nil) if l.nil?
-                # define output filename
-                output_file = "#{DATA_PATH}/searches/#{search_name}.csv" # the output file
-                raise 'Output file already exists.' if File.exists?(output_file)
-                output = File.open(output_file, 'w')
-                # parse
-                i = 0
-                source = "#{DATA_PATH}/searches/#{search_name}/*.html" # the files to be imported
-                Dir.glob(source).each do |file|
-                    doc = Nokogiri::HTML(open(file))
-                    lis = doc.xpath('//li[contains(@class, "artdeco-list__item")]')    
-                    lis.each { |li|
-                        i += 1
-                        doc2 = Nokogiri::HTML(li.inner_html)
-                        # this is where to find the full name of the lead
-                        n1 = doc2.xpath('//div[contains(@class,"artdeco-entity-lockup__title")]/a/span').first
-                        # this is where to find the name of the company, when it has a link to a linkedin company page
-                        n2 = doc2.xpath('//div[contains(@class,"artdeco-entity-lockup__subtitle")]/a').first
-                        # this is where to find the name of the company, when it has not a link to a linkedin company page
-                        company_name = nil
-                        if n2
-                            company_name = n2.text
-                        else
-                            n2 = doc2.xpath('//div[contains(@class,"artdeco-entity-lockup__subtitle")]').first 
-                            if n2
-                                company_name = n2.text.split("\n").reject { |s| s.strip.empty? }.last.strip
-                            end
-                        end
-                        # add the information to the output file
-                        line = []
-                        line << "\"#{n1.text.strip.gsub('"', '')}\"" if n1
-                        line << "\"#{company_name.strip.gsub('"', '')}\"" if company_name
-                        l.logs "#{i.to_s}, #{line.join(',')}... "
-                        output.puts line.join(',')
-                        output.flush
-                        l.done
-                    }    
-                end    
-                # close output file    
-                output.close
-            end # def self.parse_sales_navigator_result_pages(search_name)
-        end # module Parser
-
-        # return true if the domain get any random address as valid
-        def self.catch_all?(domain)
-            BlackStack::Appending.verify("008e77980535470e848a4ca859a83db0@#{domain}")
-        end
-
-        # verify an email address using the AWS IP address of our website, wich is more reliable
-        def self.verify(email)
-            url = "https://connectionsphere.com/api1.0/emails/verify.json"
-            params = {
-                :email => email,
-            }
-            res = BlackStack::Netting::call_get(url, params)
-            parsed = JSON.parse(res.body)
-            parsed['status'] == 'success'
-        end
-
-        # verify an email address
-        def self.append(fname, lname, domain)
-            ret = []
-            if !catch_all?(domain)
-                EmailVerifier.config do |config|
-                    config.verifier_email = "leandro.sardi@expandedventure.com"
-                end
-                [
-                    "#{fname}@#{domain}",
-                    "#{lname}@#{domain}",
-
-                    "#{fname}.#{lname}@#{domain}",
-                    "#{lname}.#{fname}@#{domain}",
-
-                    "#{fname}#{lname}@#{domain}",
-                    "#{lname}#{fname}@#{domain}",
-
-                    "#{fname[0]}#{lname}@#{domain}",
-                    "#{fname[0]}.#{lname}@#{domain}",
-                ].each { |email|
-                    ret << email.downcase if verify(email)
-                }
-            end
-            ret
-        end
-
-        def self.cleanup_fname(name)
-            return '' if name.nil?
-            a = name.split(/[^a-zA-Z]/)
-            a.size > 0 ? a[0] : ''
-        end
-
-        def self.cleanup_lname(name)
-            return '' if name.nil?
-            a = name.split(/[^a-zA-Z]/)
-            a.size > 1 ? a[1] : ''
-        end
-
-        def self.cleanup_company(company)
-            return '' if company.nil?
-            ret = ''
-            # stage 1: remove company-type suffixes
-            company = company.split(/ at /).last
-            company.gsub!(/LLC/, '')
-            company.gsub!(/Inc/, '')
-            company.strip! # remove leading and trailing spaces
-            # stage 2: remove LinkedIn suffixes            
-            company.gsub!(/\(\d\d\d\d - Present\)/, '')
-            company.strip! # remove leading and trailing spaces
-            # stege 3: remove non-alphanumeric characters
-            company.gsub!(/\.$/, '')  
-            company.gsub!(/\,$/, '') 
-            # stege 4: remove extra spaces
-            company.gsub!(/[^a-zA-Z0-9,\.\-\s]/, '') # remove extra spaces
-            company.strip! # remove leading and trailing spaces
-            # stage 5: choose the first part of the company name
-            company.split(' ').each { |word|
-                ret += word + ' '
-                #break if word.size >= 5 || ret.split(' ').size > 2
-                break if ret.split(' ').size > 2
-            } 
-            ret.strip!
-            # return
-            ret
-        end
-    end # Appending
-end # BlackStack
-=end