appending 0.3 → 1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 947bf74ff34b74bf14eff43808faf649b534f1d7c4939d2c5cc1215f211d922b
-  data.tar.gz: 663ba17cc651af3e83abd91b9dda7a256831d19a458ac04b46bddd721e6d6c55
+  metadata.gz: 67bdb77eef8558d6e35c4b4459f6cb9fec576cd1b0fb139b521b42c5314d8471
+  data.tar.gz: ba9f19e7c87466eabf297bc3e1f685bf13ada1ad18cfd2d346a2b22e7edfb181
 SHA512:
-  metadata.gz: d182a0b718d74367247d25e1189c39c07438b56b5fd27dba8b8c47dc96fa8db44f87e2543740856a674631217b82006bcbe4689fea1c8a2248a9b6038304a62d
-  data.tar.gz: 06acd4a0cbbea440c777b64593d89116f1d9aa370a6ebc34976b6aca868cfbbaba924494f3bb74d7d9b6668aba8b246522b8fb1c57f02a0cae3a3d29051c1b6f
+  metadata.gz: 92635a6cb6d3117e32c58b5cc372dba8f5d161655cb7d60af3f3ed3200b2bd5f948e6586238a211f62ba5dae7d26e7f06c6a9182d73c0b5a0f000c710f554933
+  data.tar.gz: 2cf1e90ca32603cdb498107016b887b251f5118522641cb6f9ef1c8fec811a0033650acd0a4511164d4672cd80d03fce317005663ed20a9f1c6f701c059a94a3

data/lib/appending.rb

@@ -1,63 +1,185 @@
 require 'csv'
+require 'email_verifier'
+require 'csv-indexer'
+require 'simple_cloud_logging'
 
 module BlackStack
     module Appending
-        # This class is used to parse the HTML files downloaded from Sales Navigator and other sources.
-        module Parser
-            # parse search results pages from sales navigator, and save the company name and full name into a CSV file
-            def self.parse_sales_navigator_result_pages(search_name, l=nil)
-                # create logger if not passed
-                l = BlackStack::DummyLogger.new(nil) if l.nil?
-                # define output filename
-                output_file = "#{DATA_PATH}/searches/#{search_name}.csv" # the output file
-                raise 'Output file already exists.' if File.exists?(output_file)
-                output = File.open(output_file, 'w')
-                # parse
-                i = 0
-                source = "#{DATA_PATH}/searches/#{search_name}/*.html" # the files to be imported
-                Dir.glob(source).each do |file|
-                    doc = Nokogiri::HTML(open(file))
-                    lis = doc.xpath('//li[contains(@class, "artdeco-list__item")]')    
-                    lis.each { |li|
-                        i += 1
-                        doc2 = Nokogiri::HTML(li.inner_html)
-                        # this is where to find the full name of the lead
-                        n1 = doc2.xpath('//div[contains(@class,"artdeco-entity-lockup__title")]/a/span').first
-                        # this is where to find the name of the company, when it has a link to a linkedin company page
-                        n2 = doc2.xpath('//div[contains(@class,"artdeco-entity-lockup__subtitle")]/a').first
-                        # this is where to find the name of the company, when it has not a link to a linkedin company page
-                        company_name = nil
-                        if n2
-                            company_name = n2.text
-                        else
-                            n2 = doc2.xpath('//div[contains(@class,"artdeco-entity-lockup__subtitle")]').first 
-                            if n2
-                                company_name = n2.text.split("\n").reject { |s| s.strip.empty? }.last.strip
-                            end
+        @@logger = nil
+        @@report = nil
+        @@indexes = []
+        @@verifier_url = 'https://connectionsphere.com/api1.0/emails/verify.json'
+        @@verifier_api_key = nil
+        @@email_fields = []
+        @@phone_fields = []
+        @@company_domain_fields = []
+
+        ## @@logger
+        def self.set_logger(logger)
+            @@logger = logger
+        end
+
+        def self.logger
+            @@logger
+        end
+
+        ## @@indexes
+        def self.add_index(index)
+            expected = [:company_name, :first_name, :last_name]
+
+            # validation: keys must be `[:company_name, :first_name, :last_name]`
+            if !index.keys.eql?(expected)
+                raise "Invalid index: #{index.keys}. Expected: #{expected}."
+            end
+            # add the index
+            @@indexes << index
+        end
+
+        def self.set_indexes(indexes)
+            @@indexes = indexes
+        end
+        
+        def self.indexes
+            @@indexes
+        end
+
+        # @@report
+        def self.report
+            @@report
+        end
+
+        # @@verifier_url
+        def self.set_verifier_url(url)
+            @@verifier_url = url
+        end
+        
+        def self.verifier_url
+            @@verifier_url
+        end
+
+        # @@verifier_api_key
+        def self.set_verifier_api_key(key)
+            @@verifier_api_key = key
+        end
+
+        def self.verifier_api_key
+            @@verifier_api_key
+        end
+
+        # @@email_fields
+        def self.set_email_fields(fields)
+            @@email_fields = fields
+        end
+
+        def self.email_fields
+            @@email_fields
+        end
+
+        # @@phone_fields
+        def self.set_phone_fields(fields)
+            @@phone_fields = fields
+        end
+
+        def self.phone_fields
+            @@phone_fields
+        end
+
+        # @@company_domain_fields
+        def self.set_company_fields(fields)
+            @@company_domain_fields = fields
+        end
+
+        def self.company_domain_fields
+            @@company_domain_fields
+        end
+
+        # set configuration
+        def self.set(h)
+            errors = []
+
+            # validation: if :indexes is present, it must be an array of objects BlackStack::CSVIndexer::Index
+            if h[:indexes]
+                if !h[:indexes].is_a?(Array)
+                    errors << "Invalid :indexes: #{h[:indexes].class}. Expected: Array."
+                else
+                    h[:indexes].each { |index|
+                        if !index.is_a?(BlackStack::CSVIndexer::Index)
+                            errors << "Invalid :indexes: #{index.class}. Expected: BlackStack::CSVIndexer::Index."
                         end
-                        # add the information to the output file
-                        line = []
-                        line << "\"#{n1.text.strip.gsub('"', '')}\"" if n1
-                        line << "\"#{company_name.strip.gsub('"', '')}\"" if company_name
-                        l.logs "#{i.to_s}, #{line.join(',')}... "
-                        output.puts line.join(',')
-                        output.flush
-                        l.done
-                    }    
-                end    
-                # close output file    
-                output.close
-            end # def self.parse_sales_navigator_result_pages(search_name)
-        end # module Parser
+                    }
+                end
+            end
+
+            # validation: if :verifier_url is present, it must be a string
+            errors << ":verifier_url must be a string." if h[:verifier_url] && !h[:verifier_url].is_a?(String)
+
+            # validation: if :verifier_api_key is present, it must be a string
+            errors << ":verifier_api_key must be a string." if h[:verifier_api_key] && !h[:verifier_api_key].is_a?(String)
+
+            # validation: if :email_fields is present, it must be an array of strings
+            if h[:email_fields]
+                if !h[:email_fields].is_a?(Array)
+                    errors << "Invalid :email_fields: #{h[:email_fields].class}. Expected: Array."
+                else
+                    h[:email_fields].each { |field|
+                        if !field.is_a?(String)
+                            errors << "Invalid :email_fields: #{field.class}. Expected: String."
+                        end
+                    }
+                end
+            end
+
+            # validation: if :phone_fields is present, it must be an array of strings
+            if h[:phone_fields]
+                if !h[:phone_fields].is_a?(Array)
+                    errors << "Invalid :phone_fields: #{h[:phone_fields].class}. Expected: Array."
+                else
+                    h[:phone_fields].each { |field|
+                        if !field.is_a?(String)
+                            errors << "Invalid :phone_fields: #{field.class}. Expected: String."
+                        end
+                    }
+                end
+            end
+
+            # validation: if :company_domain_fields is present, it must be an array of strings
+            if h[:company_domain_fields]
+                if !h[:company_domain_fields].is_a?(Array)
+                    errors << "Invalid :company_domain_fields: #{h[:company_domain_fields].class}. Expected: Array."
+                else
+                    h[:company_domain_fields].each { |field|
+                        if !field.is_a?(String)
+                            errors << "Invalid :company_domain_fields: #{field.class}. Expected: String."
+                        end
+                    }
+                end
+            end
+
+            # mapping
+            @@indexes = h[:indexes] if h[:indexes]
+            @@verifier_url = h[:verifier_url] if h[:verifier_url]
+            @@verifier_api_key = h[:verifier_api_key] if h[:verifier_api_key]
+            @@email_fields = h[:email_fields] if h[:email_fields]
+            @@phone_fields = h[:phone_fields] if h[:phone_fields]
+            @@company_domain_fields = h[:company_domain_fields] if h[:company_domain_fields]
+        end
 
         # return true if the domain get any random address as valid
+        #
+        # This is a support method for the `append` methods.
+        # The end-user should not call this method directly.
+        # 
         def self.catch_all?(domain)
             BlackStack::Appending.verify("008e77980535470e848a4ca859a83db0@#{domain}")
         end
 
         # verify an email address using the AWS IP address of our website, wich is more reliable
+        #
+        # This is a support method for the `append` methods.
+        # The end-user should not call this method directly.
+        # 
         def self.verify(email)
-            url = "https://connectionsphere.com/api1.0/emails/verify.json"
+            url = @@verifier_url
             params = {
                 :email => email,
             }
@@ -92,18 +214,24 @@ module BlackStack
             ret
         end
 
+        # This is a support method for the `append` methods.
+        # The end-user should not call this method directly.
         def self.cleanup_fname(name)
             return '' if name.nil?
             a = name.split(/[^a-zA-Z]/)
             a.size > 0 ? a[0] : ''
         end
 
+        # This is a support method for the `append` methods.
+        # The end-user should not call this method directly.
         def self.cleanup_lname(name)
             return '' if name.nil?
             a = name.split(/[^a-zA-Z]/)
             a.size > 1 ? a[1] : ''
         end
 
+        # This is a support method for the `append` methods.
+        # The end-user should not call this method directly.
         def self.cleanup_company(company)
             return '' if company.nil?
             ret = ''
@@ -131,5 +259,197 @@ module BlackStack
             # return
             ret
         end
+
+        # Find a person in the indexes by its full name and company name.
+        # Append all the information in the index row.
+        def self.find_persons_with_full_name(name, cname)
+            l = BlackStack::Appending.logger || BlackStack::DummyLogger.new
+
+            l.logs "Guessing fname from #{name}... "
+            fname = BlackStack::Appending::cleanup_fname(name)
+            l.logf fname
+
+            l.logs "Guessing lname from #{name}... "
+            lname = BlackStack::Appending::cleanup_lname(name)
+            l.logf lname
+
+            BlackStack::Appending.find_persons(fname, lname, cname)
+        end
+
+        # Find a person in the indexes by its first name, last name and company name.
+        # Append all the information in the index row.
+        def self.find_persons(fname, lname, cname)
+            l = BlackStack::Appending.logger || BlackStack::DummyLogger.new
+            h = {
+                :matches => [],
+                :enlapsed_seconds => 0,
+                :files_processed => 0,
+            }
+            # cleaning up company name
+            l.logs "Cleaning up company name #{cname}... "
+            cname = BlackStack::Appending::cleanup_company(cname)
+            l.logf cname
+            # looking for a record that matches with first name, last name and company name
+            appends = []
+            enlapsed_seconds = 0
+            files_processed = 0
+            BlackStack::Appending.indexes.each { |i|
+                l.logs "Searching into #{i.name}... "
+                ret = i.find([cname, fname, lname], false, nil)
+                # add the name of the index in the last position of the match
+                ret[:matches].each { |m| m.unshift(i.name.to_s) }
+                # add matches to the list
+                h[:matches] += ret[:matches]
+                # sum the total files and the total enlapsed seconds                
+                h[:enlapsed_seconds] += ret[:enlapsed_seconds]
+                h[:files_processed] += ret[:files_processed]
+                l.done
+            }
+            # update report
+            @@report = h
+            # return results
+            h[:matches].map { |m| BlackStack::Appending::Result.new(m) }
+        end
+
+        # Find a company in the indexes by its first name, last name and company name.
+        # Append all the information in the index row.
+        def self.find_persons_by_company(cname)
+            l = BlackStack::Appending.logger || BlackStack::DummyLogger.new
+            h = {
+                :matches => [],
+                :enlapsed_seconds => 0,
+                :files_processed => 0,
+            }
+            # looking for a record that matches with first name, last name and company name
+            appends = []
+            enlapsed_seconds = 0
+            files_processed = 0
+            BlackStack::Appending.indexes.each { |i|
+                l.logs "Searching into #{i.name}... "
+                ret = i.find([cname], true, nil)
+                # add the name of the index in the last position of the match
+                ret[:matches].each { |m| m.unshift(i.name.to_s) }
+                # add matches to the list
+                h[:matches] += ret[:matches]
+                # sum the total files and the total enlapsed seconds                
+                h[:enlapsed_seconds] += ret[:enlapsed_seconds]
+                h[:files_processed] += ret[:files_processed]
+                l.done
+            }
+            # update report
+            @@report = h
+            # return results
+            h[:matches].map { |m| BlackStack::Appending::Result.new(m) }
+        end
+
+        def self.find_verified_emails(fname, lname, cname)
+            l = BlackStack::Appending.logger || BlackStack::DummyLogger.new
+            emails = []
+            domains = []
+            verified_emails = []
+            # get lead emails from in the indexes
+            l.logs ("Searching index emails... ")
+                emails = BlackStack::Appending.find_persons(fname, lname, cname).map { |res| 
+                    res.emails 
+                }.flatten.uniq.reject { |email|
+                    email.to_s.empty?
+                }
+            l.done
+            # get company domains from the indexes
+            l.logs ("Searching index domains... ")
+                domains = BlackStack::Appending.find_persons_by_company(cname).map { |res| 
+                    res.company_domains
+                }.flatten.reject { |email|
+                    email.to_s.empty?
+                }.map { |domain|
+                    # normalize domain
+                    domain.to_s.gsub('www.', '').downcase
+                }.uniq
+            l.done
+            # verify all the emails found in the indexes
+            l.logs ("Verifying index emails... ")
+            emails.each { |email|
+                l.logs "Verifying #{email}... "
+                domain = email.split('@').last
+                verified_emails << email if BlackStack::Appending.verify(email) && !BlackStack::Appending.catch_all?(domain)
+                l.done
+            }
+            l.done
+            # appending with domains found in the indexes
+            l.logs ("Appending with domains... ")
+            domains.each { |domain|
+                l.logs "Appending with #{domain}... "
+                verified_emails += BlackStack::Appending.append(fname, lname, domain)
+                l.done
+            }
+            l.done
+            # return 
+            verified_emails.uniq
+        end
+
+        # 
+        class Result
+            # array of values.
+            # first 3 values are index name, key and row-number.
+            attr_accessor :match
+            
+            def initialize(a)
+                self.match = a
+            end
+
+            # From a given match (with the name of its index in the first position), get the value of a field by its name.
+            def value(field)
+                # get the index_name
+                index_name = match[0]
+                # get the index descriptor
+                index = BlackStack::CSVIndexer.indexes.select { |i| i.name == index_name }.first
+                # get position of the field into the hash descriptior
+                k = index.mapping.to_a.map { |m| m[0].to_s }.index(field.to_s)
+                # return nil if the field is not found
+                return nil if k.nil?
+                # get the field value
+                match[k+3].to_s
+            end
+
+            # Call value() method.
+            def val(field)
+                self.value(field)
+            end
+
+            # From a given match (with the name of its index in the first position), get the email addresses.
+            def emails()
+                keys = BlackStack::Appending.email_fields
+                ret = []
+                keys.each { |k|
+                    v = self.value(k)
+                    ret << v if v
+                }
+                ret
+            end
+
+            # From a given match (with the name of its index in the first position), get the phone numbers.
+            def phones()
+                keys = BlackStack::Appending.phone_fields
+                ret = []
+                keys.each { |k|
+                    v = self.value(k)
+                    ret << v if v
+                }
+                ret
+            end
+
+            # From a given match (with the name of its index in the first position), get the company domains.
+            def company_domains()
+                keys = BlackStack::Appending.company_domain_fields
+                ret = []
+                keys.each { |k|
+                    v = self.value(k)
+                    ret << v if v
+                }
+                ret
+            end
+
+        end # class Result
+
     end # Appending
 end # BlackStack

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: appending
 version: !ruby/object:Gem::Version
-  version: '0.3'
+  version: '1.0'
 platform: ruby
 authors:
 - Leandro Daniel Sardi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-12-19 00:00:00.000000000 Z
+date: 2022-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csv
@@ -30,6 +30,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.2.2
+- !ruby/object:Gem::Dependency
+  name: email_verifier
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 - !ruby/object:Gem::Dependency
   name: blackstack-core
   requirement: !ruby/object:Gem::Requirement