navi_client 1.3.1 → 1.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8d9bbc833c46fc05429522a82871694883ec8b76abbed13202b0d7541eca7761
-  data.tar.gz: 6e24066223a2a3b810dcc95dc660e90717da6d6ec02ed9fcf45383d9be1ad152
+  metadata.gz: b7810ed026c2d49433d2b0e25ac0fcd49cd2c15823bd44b2308d18c44d462f02
+  data.tar.gz: d24ddc1054dbdfc6c8c20ed2a458c9b2240a216caf5f9c775a8ec2d4adbede40
 SHA512:
-  metadata.gz: 98d021f7c375641d79bcb37f2e2ab319a589800e158db453299b082ca1b8f98ed44569c094ca32709b3ef751d6db71ca7ed6ff3226fdbd4fdff577fcd21c73ca
-  data.tar.gz: 5e941373fde75fc2018b5a4d61c34c87be675db480468f665c4ccc52b027091c739765df7aa651fa9d47ccf4f0e0b19381269f0e1c9f55ac83bcbbc400d31074
+  metadata.gz: ad4631fd864a936100708cdd5b66c25416ba0d47e6b7ffb65ffcdc0c4767038854820aa0369e7608df5483f22e6976e19c00b4e87b76f0b1b95c4431d322cd13
+  data.tar.gz: 96c3c79eced0d78cf52ed03585f4ee3478ee1938fbeabadeca1533577c6958b8394e7e5ce0929b04941f93fc8545f4cf05044986362f78469a7204603820d461

data/.gitignore

@@ -7,3 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+.idea/

data/README.md

@@ -4,6 +4,12 @@ Welcome to your new gem! In this directory, you'll find the files you need to be
 
 TODO: Delete this and the text above, and describe your gem
 
+## Generate Documentation
+```
+$ rdoc
+```
+Browse html file inside the ```doc``` folder
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -38,4 +44,3 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/[USERN
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
-

data/lib/client.rb

@@ -12,7 +12,14 @@ require "httparty"
 require "http_service/naviai"
 require 'openssl'
 
+##
+# This module provides the common functionality that will be needed for local and cloud module.
+# @client_type defines if the gem is being used for local-lockbox or cloud-version.
+#
 module Client
+
+  @logglyTag
+
   def logger
     @logger
   end
@@ -71,26 +78,31 @@ module Client
     # https://stackoverflow.com/questions/16516464/read-gmail-xoauth-mails-without-marking-it-read
     imap.examine folder
 
-    message_ids = imap.search(search_condition)
+    message_ids = imap.uid_search(search_condition)
 
-    if @debug
-      if message_ids.empty?
-        @logger.debug "\nno messages found.\n"
-        return
-      else
-        @logger.debug "\nProcessing #{message_ids.count} mails.\n"
-      end
+    meta_dir = @download_path + config['username'] + '/meta/'
+    message_ids_saved = File.directory?(meta_dir) || @client_type == "cloud" ? (Dir.entries meta_dir).map { |i| i.split("_").first.to_i } : []
+
+    message_ids = message_ids - message_ids_saved
+
+    if message_ids.empty?
+      @logger.info "No new emails found..."
+    elsif message_ids_saved.any?
+      @logger.info "Found emails saved in your #{@client_type}. Downloading only #{message_ids.count} new emails..."
+    else
+      @logger.info "Downloading #{message_ids.count} emails."
     end
+
     message_ids.each_with_index do |message_id, i|
       # fetch all the email contents
-      data = imap.fetch(message_id, "RFC822")
+      data = imap.uid_fetch(message_id, "RFC822")
       data.each do |d|
         msg = d.attr['RFC822']
         # instantiate a Mail object to avoid further IMAP parameters nightmares
         mail = Mail.read_from_string msg
 
         # call the block with mail object as param
-        process_email_block.call mail
+        process_email_block.call mail, i, i == message_ids.length-1, message_id
 
         # mark as read
         if @mark_as_read
@@ -100,7 +112,12 @@ module Client
     end
   end
 
-  def process_email(mail)
+
+  ##
+  # Process each email downloaded from imap-server and creates meta file that will be sent over to the navi-ai service.
+  # Meta will content information like: ['from', 'to', 'cc', ..., 'body_url']. This information is then used by navi-ai to parse the body content.
+  #
+  def process_email(mail, uid)
     meta = Hash.new
     custom_uid = (Time.now.to_f * 1000).to_s + "_" + mail.__id__.to_s
 
@@ -108,7 +125,13 @@ module Client
 
     invalid_cc_email = mail.cc.nil? || !mail.cc.is_a?(Array)
 
-    meta["from"] = mail.from.first unless mail.from.nil?
+    unless mail.from.nil?
+      if defined? mail.from.first
+        meta["from"] = mail.from.first
+      else
+        meta["from"] = mail.from
+      end
+    end
     meta["to"] = invalid_to_email ? mail.to : mail.to.join(";")
     meta["cc"] = mail.cc.join(";") unless invalid_cc_email
     meta["subject"] = mail.subject
@@ -124,7 +147,7 @@ module Client
       meta.merge!(m) unless m.nil?
     end
 
-    save(meta, "meta/#{custom_uid}")
+    save(meta, "meta/#{uid.to_s + '_' + custom_uid}")
   end
 
   def encrypt(data)
@@ -163,6 +186,9 @@ module Client
     Time.now.utc.iso8601(3)
   end
 
+  ##
+  # If the gem is being used for local-lockbox mode, after fetching all emails, the process will go idle mode. If user send the interrupt command, it will call shutdown to disconnect the connection with imap server
+  #
   def shutdown(imap)
     imap.idle_done
     imap.logout unless imap.disconnected?
@@ -171,4 +197,27 @@ module Client
     @logger.info "#{$0} has ended (crowd applauds)"
     exit 0
   end
+
+  def logToLoggly(messageBody)
+
+    if(@logglyTag)
+      begin
+        HTTParty.post("http://logs-01.loggly.com/bulk/0d67f93b-6568-4b00-9eca-97d0ea0bd5a1/tag/#{@logglyTag}/",
+                      body: messageBody.to_json,
+                      headers: { 'Content-Type' => 'application/json' } )
+      rescue
+        true
+      end
+
+    else
+      @logger.info "Logging to Loggly disabled"
+      @logger.info messageBody
+    end
+
+  end
+
+  def setupLoggly(tag)
+    @logglyTag = tag
+  end
+
 end

data/lib/cloud/navi_cloud_client.rb

@@ -1,6 +1,12 @@
 require Gem::Specification.find_by_name("navi_client").gem_dir+"/lib/client"
+require 'concurrent'
 
 module NaviClient
+
+  ##
+  # This class represents the client for cloud version.
+  # In cloud version, all the content will be saved and also being used from the s3
+  #
   class Cloud
     include Concurrent::Async
     include Client
@@ -31,6 +37,8 @@ module NaviClient
       # client_type
       @client_type = "cloud"
 
+      @download_path = config[:s3_download_folder] + '/'
+
       # set email_address of current_user for s3 folder name
       @current_user_email = nil
     end
@@ -52,6 +60,9 @@ module NaviClient
       @current_user_email = email
     end
 
+    ##
+    # send bulk request to navi-ai service with list of input files downloaded from imap server.
+    #
     def send_request(in_filenames = [])
       unless in_filenames.blank?
         download_path = config['s3_download_folder'] + "/" + @current_user_email
@@ -62,6 +73,9 @@ module NaviClient
       end
     end
 
+    ##
+    # Downloads the email content from imap-server and save it to the download_path
+    #
     def download(message, custom_uid)
       download_path = config[:s3_download_folder] + "/" + @current_user_email
       if ['text/plain', 'text/html'].include? message.mime_type
@@ -77,6 +91,10 @@ module NaviClient
       end
     end
 
+    ##
+    # save data to download_path with file named by filename params.
+    # Input is hashmap, and it save the hashmap as yml format.
+    #
     def save(data={}, filename)
       download_path = config[:s3_download_folder] + "/" + @current_user_email
       filepath = download_path + "/" + filename + ".yml"
@@ -84,6 +102,8 @@ module NaviClient
       return upload_to_s3(filepath, data.to_yaml)
     end
 
+    ##
+    # Helper function to upload the content to the s3
     def upload_to_s3(file_path, content)
       credentials = Aws::Credentials.new(config[:aws_key], config[:aws_secret])
       s3 = Aws::S3::Client.new(credentials: credentials, region: config[:aws_region])

data/lib/local/navi_local_client.rb

@@ -1,6 +1,11 @@
 require_relative '../client'
 
 module NaviClient
+
+  ##
+  # This class represents the client for local lockbox mode.
+  # In local-lockbox mode, all the content will be saved and also being used from the local file system
+  #
   class Local
     include Client
     CONFIG_PATH = File.join(ENV['HOME'], '/.navi/config.yml')
@@ -24,6 +29,8 @@ module NaviClient
       # authentication token received from sso_web used to authenticate the request to database_api
       @token = nil
 
+      @download_path = config['download_path']
+
       # client_type
       @client_type = "local"
     end
@@ -46,8 +53,11 @@ module NaviClient
       @token = "Token: #{token}##{config['username']}"
     end
 
+    ##
+    # Downloads the email content from imap-server and save it to the download_path
+    #
     def download(message, custom_uid)
-      download_path = config['download_path']
+      download_path = config['download_path'] + config['username'] + '/'
       if ['text/plain', 'text/html'].include? message.mime_type
 
         h = Hash.new
@@ -62,8 +72,12 @@ module NaviClient
       end
     end
 
+    ##
+    # save data to download_path with file named by filename params.
+    # Input is hashmap, and it save the hashmap as yml format.
+    #
     def save(data={}, filename)
-      download_path = config['download_path']
+      download_path = config['download_path'] + config['username'] + '/'
       filepath = download_path + filename + ".yml"
 
       mkdir_if_not_exist(filepath)
@@ -72,10 +86,13 @@ module NaviClient
       return filepath
     end
 
+    ##
+    # send bulk request to navi-ai service with list of input files downloaded from imap server.
+    #
     def send_request(in_filenames = [])
       unless in_filenames.empty?
         download_path = config['download_path']
-        filename = download_path + "inputs/" + (Time.now.to_f * 1000).to_s
+        filename = download_path + config['username'] + "/inputs/" + (Time.now.to_f * 1000).to_s
 
         mkdir_if_not_exist(filename)
 
@@ -122,7 +139,7 @@ module NaviClient
           # We're out, which means there are some emails ready for us.
           # Go do a search for UNSEEN and fetch them.
           filenames = []
-          retrieve_emails(imap, search_condition, folder) { |mail| filenames << process_email(mail)}
+          retrieve_emails(imap, search_condition, folder) { |mail, i, isLast, id| filenames << process_email(mail, id)}
           self.send_request(filenames)
 
           HTTPService::NaviAI.generate_csv(username)

data/lib/navi_client.rb

@@ -1,6 +1,9 @@
 require "local/navi_local_client"
 require "cloud/navi_cloud_client"
 
+##
+# This module represents the client that helps connection between navi and the imap server.
+#
 module NaviClient
 
 end

data/lib/navi_client/version.rb

@@ -1,3 +1,3 @@
 module NaviClient
-  VERSION = "1.3.1"
+  VERSION = "1.3.3"
 end

data/navi_client.gemspec

@@ -32,4 +32,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_runtime_dependency "httparty"
   spec.add_runtime_dependency "mail"
+  spec.add_runtime_dependency "concurrent-ruby", "~> 1.0.5"
 end

metadata

@@ -1,87 +1,101 @@
 --- !ruby/object:Gem::Specification
 name: navi_client
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.3.3
 platform: ruby
 authors:
 - Sujit
 - Surya
 - Saroj
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-02-19 00:00:00.000000000 Z
+date: 2018-03-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.11'
-  name: bundler
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.11'
 - !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
-  name: rake
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-  name: rspec
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
+  name: httparty
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: httparty
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: mail
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: mail
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.5
 description: " Write a longer description or delete this line."
 email:
 - sujit@whitehatengineering.com
@@ -113,7 +127,7 @@ homepage: http://navihq.com
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -128,9 +142,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.11
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.7.4
+signing_key: 
 specification_version: 4
 summary: Write a short summary, because Rubygems requires one.
 test_files: []