mass-client 1.0.1

data/lib/base-line/source.rb

@@ -0,0 +1,187 @@
+module Mass
+    class Source < BlackStack::Base
+        attr_accessor :type
+
+        def self.object_name
+            'source'
+        end
+
+        def initialize(h)
+            super(h)
+            self.type = Mass::SourceType.new(h['source_type_desc']).child_class_instance
+        end
+
+        # convert the source_type into the ruby class to create an instance.
+        # example: Apollo --> Mass::ApolloAPI
+        def class_name_from_source_type
+            source_type = self.desc['source_type']
+            "Mass::#{source_type}" 
+        end
+
+        # crate an instance of the profile type using the class defined in the `desc['name']` attribute.
+        # override the base method
+        def child_class_instance
+            source_type = self.desc['source_type']
+            key = self.class_name_from_source_type
+            raise "Source code of souurce type #{source_type} not found. Create a class #{key} in the folder `/lib` of your mass-sdk." unless Kernel.const_defined?(key)
+            ret = Kernel.const_get(key).new(self.desc)
+            return ret
+        end
+
+        # If the profile `access` is not `:rpa`, raise an exception.
+        # Return `true` if the `url` is valid.
+        # Return `false` if the `url` is not valid. 
+        #
+        # Overload this method in the child class.
+        # 
+        def valid_source_url?(url:)
+            # If the profile `access` is not `:rpa`, raise an exception.
+            raise "The method `valid_source_url?` is not allowed for #{self.profile_type.desc['access'].to_s} access." if self.profile_type.desc['access'] != :rpa
+            # Return `true` if the `url` is valid.
+            # Return `false` if the `url` is not valid. 
+            true
+        end
+
+        # Return the same URL in a normalized form:
+        # - remove all GET parameters.
+        # - remove all trailing slashes.
+        #
+        # If the profile `access` is not `:rpa`, raise an exception.
+        # If the `url` is not valid, raise an exception.
+        # Return the normalized URL.
+        #
+        # Overload this method in the child class.
+        #
+        def normalized_source_url(url:)
+            # If the profile `access` is not `:rpa`, raise an exception.
+            raise "The method `normalized_source_url` is not allowed for #{self.profile_type.desc['access'].to_s} access." if self.profile_type.desc['access'] != :rpa
+            # If the `url` is not valid, raise an exception.
+            raise "The URL is not valid." if !self.valid_source_url?(url: url)
+            # Return the same URL in a normalized form:
+            # - remove all GET parameters.
+            # - remove all trailing slashes.
+            url = url.gsub(/\?.*$/, '').strip
+            url = ret.gsub(/\/+$/, '')
+            # Return the normalized URL.
+            url
+        end
+
+        # If the profile `access` is not `:api`, raise an exception.
+        # Parameter `params` must be a hash.
+        # Return `true` if the `params` are valid.
+        # Return `false` if the `params` are not valid. 
+        def valid_source_params?(params:)
+            # If the profile `access` is not `:api`, raise an exception.
+            raise "The method `valid_source_params?` is not allowed for #{self.profile_type.desc['access'].to_s} access." if self.profile_type.desc['access'] != :api
+            # Parameter `params` must be a hash.
+            raise "The parameter `params` must be a hash." if !params.is_a?(Hash)
+            # Return `true` if the `params` are valid.
+            # Return `false` if the `params` are not valid. 
+            true
+        end
+
+        # return array of event elements
+        def event_elements(job:)
+          raise "The method `event_elements` is not implemented for #{self.class.name}."
+        end
+
+        # scroll down the page until N event elements are showed up
+        def show_up_event_elements(job:, event_limit:, max_scrolls:, logger:nil)
+              l = logger || BlackStack::DummyLogger.new(nil)
+              driver = job.profile.driver
+              # scroll down
+              i = 0
+              prev_n_events = 0
+              security_height = 150  
+              lis = self.event_elements(job: job)
+              n_events = lis.size
+              while (i<max_scrolls || n_events>prev_n_events) && n_events<event_limit
+                i += 1
+
+                prev_n_events = n_events
+                lis = self.event_elements(job: job)
+                n_events = lis.size
+  
+                # scroll down the exact height of the viewport
+                # reference: https://stackoverflow.com/questions/1248081/how-to-get-the-browser-viewport-dimensions
+                l.logs "Scrolling down (#{i.to_s.blue}/#{max_scrolls.to_s.blue} - #{n_events.to_s.blue}/#{event_limit.to_s.blue} events showed up)... "
+                step = self.desc['scrolling_step'] + rand(self.desc['scrolling_step_random'].to_i)
+                driver.execute_script("window.scrollTo(0, #{i.to_s}*#{step})")
+                #driver.execute_script("window.scrollTo(0, #{i.to_s}*(Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0)-#{security_height}))")
+                sleep(5)
+                l.logf "done".green
+
+                # screenshot
+                l.logs 'Screenshot... '
+                job.desc['screenshots'] << job.profile.screenshot
+                l.logf 'done'.green + " (#{job.desc['screenshots'].size.to_s.blue} total)"
+              end
+        end
+
+        # Return a hash desriptor of the events found.
+        #
+        # Parameters:
+        # - If the profile `access` is `:rpa`, then the `bot_driver` parameter is mandatory.
+        # - If the profile `access` is `:api`, then the `api_key` parameter is mandatory.
+        #
+        # - If the profile `access` is `:mta`, raise an exception.
+        #
+        # - If the profile `access` is `:rpa`, then the `bot_url` parameter is mandatory, and it must be a valid URL.
+        # - If the profile `access` is `:api`, then the `api_params` parameter is mandatory and it must be a hash.
+        # 
+        # - The `event_count` is for scrolling down (or perform any other required action) until finding `event_count` events.
+        #
+        # Output:
+        # {
+        #    'status' => :performed, # if it is not 'success', then it is an error description.
+        #    'snapshot' => 'https://foo.com/snapshot.png'
+        #    'screenshots' => [
+        #        # array of URLs to screenshots
+        #    ],
+        #    'events' => [
+        #        'url' => 'https://facebook.com/john-doe/posts/12345', # normalized URL of the event
+        #        'title' => 'Join my Facebook Community!'
+        #        'content' => 'My name is John Doe and I invite everyone to join my Facebook Community: facebook.com/groups/john-doe-restaurants!',
+        #        'pictures' => [
+        #            # array of URLs to pictures scraped from the post and uploaded to our DropBox.
+        #        ],
+        #        'lead' => {
+        #            'name' => 'John Doe',
+        #            'url' => 'https://facebook.com/john-doe',
+        #            'headline' => "Founder & CEO at Doe's Restaurants", 
+        #            'picture' => 'https://foo.com/john-doe.png'
+        #        }
+        #    ],
+        # }
+        # 
+        def do(job:, logger:nil)
+            # If the profile `access` is `:rpa`, then the `bot_driver` parameter is mandatory.
+            
+            #raise "The parameter `bot_driver` is mandatory." if bot_driver.nil? if self.profile_type.desc['access'].to_sym == :rpa
+            # If the profile `access` is `:api`, then the `api_key` parameter is mandatory.
+            #raise "The parameter `api_key` is mandatory." if api_key.nil? if self.profile_type.desc['access'].to_sym == :api
+            # If the profile `access` is `:mta`, raise an exception.
+            raise "The method `do` is not allowed for #{self.profile_type.desc['access'].to_s} access." if self.profile_type.desc['access'].to_sym == :mta
+            # If the profile `access` is `:rpa`, then the `bot_url` parameter is mandatory, and it must be a valid URL.
+            #raise "The parameter `bot_url` is mandatory." if bot_url.nil? if self.profile_type.desc['access'].to_sym == :rpa
+            # If the profile `access` is `:api`, then the `api_params` parameter is mandatory and it must be a hash.
+            #raise "The parameter `api_params` is mandatory." if api_params.nil? if self.profile_type.desc['access'].to_sym == :api
+            # The `event_count` is for scrolling down (or perform any other required action) until finding `event_count` events.
+            #raise "The parameter `event_count` must be an integer higher or equal then 0." if !event_count.is_a?(Integer) || event_count < 0
+
+            # return
+            return {
+                'status' => :performed, # if it is not 'success', then it is an error description.
+                'screenshots' => [
+                    # array of URLs to screenshots
+                ],
+                # array of URLs to HTML snapshots
+                'snapshot_url' => nil,
+                'events' => [
+                    # array of event descriptors
+                ],
+            }
+        end # def do
+
+    end # class Source
+end # module Mass

data/lib/base-line/source_type.rb

@@ -0,0 +1,24 @@
+module Mass
+    class SourceType < BlackStack::Base
+        # NEVER EVER DOWNLOAD LINKEDIN/FACEBOOK IMAGES USING THE LOCAL IP ADDRESS.
+        # USE THE SAME BROWSER DRIVER AND JAVASCRIPT INSTEAD OF THE LOCAL IP ADDRESS.
+        #extend BlackStack::Storage # include the module with the class methods
+
+        attr_accessor :profile_type
+
+        def initialize(h={})
+            super(h)
+            self.profile_type = Mass::ProfileType.page(
+                page: 1,
+                limit: 1,
+                filters: {
+                    name: h['profile_type']
+                }
+            ).first.child_class_instance if h['profile_type']
+        end
+
+        def self.object_name
+            'source_type'
+        end
+    end # class SourceType
+end # module Mass

data/lib/base-line/tag.rb

@@ -0,0 +1,7 @@
+module Mass
+    class Tag < BlackStack::Base
+        def self.object_name
+            'tag'
+        end
+    end # class Tag
+end # module Mass

data/lib/first-line/profile_api.rb

@@ -0,0 +1,7 @@
+module Mass
+    class ProfileAPI < Mass::Profile
+
+        # TODO: Code Me!
+
+    end # class ProfileAPI
+end # module Mass

data/lib/first-line/profile_mta.rb

@@ -0,0 +1,147 @@
+module Mass
+    class ProfileMTA < Mass::Profile
+
+        # Scrape the inbox of the profile.
+        # Return a an array of hash descriptors of outreach records.
+        # 
+        # Parameters:
+        # - limit: the maximum number of messages to scrape. Default: 100.
+        # - only_unread: if true, then only the unread messages will be scraped. This parameter is not used by :mta profiles. Default: true.
+        # - logger: a logger object to log the process. Default: nil.
+        #
+        # Example of a hash descritor into the returned array:
+        # ```
+        # {
+        #    # a scraped message is always a :performed message
+        #    'status' => :performed,
+        #    # what is the outreach type?
+        #    # e.g.: :LinkedIn_DirectMessage
+        #    # decide this in the child class.
+        #    'outreach_type' => nil,
+        #    # hash descriptor of the profile who is scraping the inbox
+        #    'profile' => self.desc,
+        #    # hash descriptor of the lead who is the conversation partner
+        #    'lead' => nil,
+        #    # if the message has been sent by the profile, it is :outgoing.
+        #    # if the message has been sent by the lead, it is :incoming.
+        #    'direction' => nil, 
+        #    # the content of the message
+        #    'subject' => nil,
+        #    'body' => nil,
+        # }
+        # ```
+        #
+        def inboxcheck(limit: 100, only_unread:true, logger:nil)
+            l = logger || BlackStack::DummyLogger.new(nil)
+            ret = []
+            p = self
+            t = self.type
+            sources = [
+                {:folder=>p.desc['inbox_label'] || t.desc['default_inbox_label'], :track_field=>'imap_inbox_last_id'}, 
+                #{:folder=>p.desc['spam_label'] || t.desc['default_inbox_label'], :track_field=>'imap_spam_last_id'},
+            ]
+
+            # connecting imap 
+            l.logs "Connecting IMAP... "
+            imap = Net::IMAP.new(
+                p.desc['imap_address'] || t.desc['default_imap_address'], 
+                p.desc['imap_port'] || t.desc['default_imap_port'], 
+                true
+            )
+            conn = imap.login(
+                p.desc['imap_username'], 
+                p.desc['imap_password']
+            )
+            l.logf "done (#{conn.name})"
+
+            sources.each { |source|
+                folder = source[:folder]
+                track_field = source[:track_field]
+
+                l.logs "Choosing mailbox #{folder}... "
+                    l.logs "Examine folder... "
+                    res = imap.examine(folder)
+                    l.logf "done (#{res.name})"
+            
+                    # Gettin latest `limit` messages received, in descendent order (newer first), 
+                    # in order to stop when I find the latest procesed before.
+                    l.logs "Getting latest #{limit.to_s} messages... "
+                    ids = imap.search(["SUBJECT", p.desc['search_all_wildcard']]).reverse[0..limit]
+                    l.logf "done (#{ids.size.to_s} messages)"
+
+                    # iterate the messages
+                    last_message_id = nil
+                    ids.each { |id|
+                        l.logs "Processing message #{id.to_s.blue}... "
+                        # getting the envelope
+                        envelope = imap.fetch(id, "ENVELOPE")[0].attr["ENVELOPE"]
+
+                        # TODO: develop a normalization function for mail.message_id
+                        message_id = envelope.message_id.to_s.gsub(/^</, '').gsub(/>$/, '')
+                        
+                        # if this is the first message, then remember it
+                        last_message_id = message_id if last_message_id.nil?
+
+                        # check if this message_id is the latest processed
+                        if message_id == p.desc[track_field]
+                            l.logf "skip".yellow + " (already processed)"
+                            break
+                        #elsif envelope.subject =~ /[0-9A-Z]{7}\-[0-9A-Z]{7}/i
+                        #    l.logf "Instantly warming email".red
+                        else
+                            lead_email = envelope.from[0].mailbox.to_s + '@' + envelope.from[0].host.to_s
+                            lead_name = envelope.from[0].name
+                            subject = envelope.subject
+                            body = imap.fetch(id, "BODY[]")[0].attr["BODY[]"]
+                            
+                            # analyzing bounce reports
+                            #rep = Sisimai.make(body)
+                            #is_bounce = !rep.nil?
+                            #bounce_reason = rep[0].reason if rep
+                            #bounce_diagnosticcode = rep[0].diagnosticcode if rep
+            
+                            h = {
+                                # a scraped message is always a :performed message
+                                'status' => :performed,
+                                # what is the outreach type?
+                                # e.g.: :LinkedIn_DirectMessage
+                                # decide this in the child class.
+                                'outreach_type' => :GMail_DirectMessage,
+                                # hash descriptor of the profile who is scraping the inbox
+                                'profile' => p.desc,
+                                # hash descriptor of the lead who is the conversation partner
+                                'lead_or_company' => {
+                                  'name' => lead_name,
+                                  'email' => lead_email,
+                                },
+                                # if the message has been sent by the profile, it is :outgoing.
+                                # if the message has been sent by the lead, it is :incoming.
+                                'direction' => :incoming, 
+                                # the content of the message
+                                'subject' => subject,
+                                'body' => body,
+                            }
+                            ret << h
+
+                            l.logf "done".green
+                        end                        
+                    }
+                    
+                    # remember the latest message_id processed
+                    p.desc[track_field] = last_message_id if last_message_id
+
+                l.done
+
+            } # end folders.each
+
+            # disconnect
+            l.logs "Disconnecting IMAP... "
+            res = imap.logout
+            l.logf "done (#{res.name})"
+
+            # return
+            return ret
+        end # def inboxcheck
+        
+    end # class ProfileMTA
+end # module Mass