navi_client 1.3.1 → 1.3.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/README.md +6 -1
- data/lib/client.rb +62 -13
- data/lib/cloud/navi_cloud_client.rb +20 -0
- data/lib/local/navi_local_client.rb +21 -4
- data/lib/navi_client.rb +3 -0
- data/lib/navi_client/version.rb +1 -1
- data/navi_client.gemspec +1 -0
- metadata +31 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b7810ed026c2d49433d2b0e25ac0fcd49cd2c15823bd44b2308d18c44d462f02
|
4
|
+
data.tar.gz: d24ddc1054dbdfc6c8c20ed2a458c9b2240a216caf5f9c775a8ec2d4adbede40
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ad4631fd864a936100708cdd5b66c25416ba0d47e6b7ffb65ffcdc0c4767038854820aa0369e7608df5483f22e6976e19c00b4e87b76f0b1b95c4431d322cd13
|
7
|
+
data.tar.gz: 96c3c79eced0d78cf52ed03585f4ee3478ee1938fbeabadeca1533577c6958b8394e7e5ce0929b04941f93fc8545f4cf05044986362f78469a7204603820d461
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -4,6 +4,12 @@ Welcome to your new gem! In this directory, you'll find the files you need to be
|
|
4
4
|
|
5
5
|
TODO: Delete this and the text above, and describe your gem
|
6
6
|
|
7
|
+
## Generate Documentation
|
8
|
+
```
|
9
|
+
$ rdoc
|
10
|
+
```
|
11
|
+
Browse html file inside the ```doc``` folder
|
12
|
+
|
7
13
|
## Installation
|
8
14
|
|
9
15
|
Add this line to your application's Gemfile:
|
@@ -38,4 +44,3 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/[USERN
|
|
38
44
|
## License
|
39
45
|
|
40
46
|
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
41
|
-
|
data/lib/client.rb
CHANGED
@@ -12,7 +12,14 @@ require "httparty"
|
|
12
12
|
require "http_service/naviai"
|
13
13
|
require 'openssl'
|
14
14
|
|
15
|
+
##
|
16
|
+
# This module provides the common functionality that will be needed for local and cloud module.
|
17
|
+
# @client_type defines if the gem is being used for local-lockbox or cloud-version.
|
18
|
+
#
|
15
19
|
module Client
|
20
|
+
|
21
|
+
@logglyTag
|
22
|
+
|
16
23
|
def logger
|
17
24
|
@logger
|
18
25
|
end
|
@@ -71,26 +78,31 @@ module Client
|
|
71
78
|
# https://stackoverflow.com/questions/16516464/read-gmail-xoauth-mails-without-marking-it-read
|
72
79
|
imap.examine folder
|
73
80
|
|
74
|
-
message_ids = imap.
|
81
|
+
message_ids = imap.uid_search(search_condition)
|
75
82
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
+
meta_dir = @download_path + config['username'] + '/meta/'
|
84
|
+
message_ids_saved = File.directory?(meta_dir) || @client_type == "cloud" ? (Dir.entries meta_dir).map { |i| i.split("_").first.to_i } : []
|
85
|
+
|
86
|
+
message_ids = message_ids - message_ids_saved
|
87
|
+
|
88
|
+
if message_ids.empty?
|
89
|
+
@logger.info "No new emails found..."
|
90
|
+
elsif message_ids_saved.any?
|
91
|
+
@logger.info "Found emails saved in your #{@client_type}. Downloading only #{message_ids.count} new emails..."
|
92
|
+
else
|
93
|
+
@logger.info "Downloading #{message_ids.count} emails."
|
83
94
|
end
|
95
|
+
|
84
96
|
message_ids.each_with_index do |message_id, i|
|
85
97
|
# fetch all the email contents
|
86
|
-
data = imap.
|
98
|
+
data = imap.uid_fetch(message_id, "RFC822")
|
87
99
|
data.each do |d|
|
88
100
|
msg = d.attr['RFC822']
|
89
101
|
# instantiate a Mail object to avoid further IMAP parameters nightmares
|
90
102
|
mail = Mail.read_from_string msg
|
91
103
|
|
92
104
|
# call the block with mail object as param
|
93
|
-
process_email_block.call mail
|
105
|
+
process_email_block.call mail, i, i == message_ids.length-1, message_id
|
94
106
|
|
95
107
|
# mark as read
|
96
108
|
if @mark_as_read
|
@@ -100,7 +112,12 @@ module Client
|
|
100
112
|
end
|
101
113
|
end
|
102
114
|
|
103
|
-
|
115
|
+
|
116
|
+
##
|
117
|
+
# Process each email downloaded from imap-server and creates meta file that will be sent over to the navi-ai service.
|
118
|
+
# Meta will content information like: ['from', 'to', 'cc', ..., 'body_url']. This information is then used by navi-ai to parse the body content.
|
119
|
+
#
|
120
|
+
def process_email(mail, uid)
|
104
121
|
meta = Hash.new
|
105
122
|
custom_uid = (Time.now.to_f * 1000).to_s + "_" + mail.__id__.to_s
|
106
123
|
|
@@ -108,7 +125,13 @@ module Client
|
|
108
125
|
|
109
126
|
invalid_cc_email = mail.cc.nil? || !mail.cc.is_a?(Array)
|
110
127
|
|
111
|
-
|
128
|
+
unless mail.from.nil?
|
129
|
+
if defined? mail.from.first
|
130
|
+
meta["from"] = mail.from.first
|
131
|
+
else
|
132
|
+
meta["from"] = mail.from
|
133
|
+
end
|
134
|
+
end
|
112
135
|
meta["to"] = invalid_to_email ? mail.to : mail.to.join(";")
|
113
136
|
meta["cc"] = mail.cc.join(";") unless invalid_cc_email
|
114
137
|
meta["subject"] = mail.subject
|
@@ -124,7 +147,7 @@ module Client
|
|
124
147
|
meta.merge!(m) unless m.nil?
|
125
148
|
end
|
126
149
|
|
127
|
-
save(meta, "meta/#{custom_uid}")
|
150
|
+
save(meta, "meta/#{uid.to_s + '_' + custom_uid}")
|
128
151
|
end
|
129
152
|
|
130
153
|
def encrypt(data)
|
@@ -163,6 +186,9 @@ module Client
|
|
163
186
|
Time.now.utc.iso8601(3)
|
164
187
|
end
|
165
188
|
|
189
|
+
##
|
190
|
+
# If the gem is being used for local-lockbox mode, after fetching all emails, the process will go idle mode. If user send the interrupt command, it will call shutdown to disconnect the connection with imap server
|
191
|
+
#
|
166
192
|
def shutdown(imap)
|
167
193
|
imap.idle_done
|
168
194
|
imap.logout unless imap.disconnected?
|
@@ -171,4 +197,27 @@ module Client
|
|
171
197
|
@logger.info "#{$0} has ended (crowd applauds)"
|
172
198
|
exit 0
|
173
199
|
end
|
200
|
+
|
201
|
+
def logToLoggly(messageBody)
|
202
|
+
|
203
|
+
if(@logglyTag)
|
204
|
+
begin
|
205
|
+
HTTParty.post("http://logs-01.loggly.com/bulk/0d67f93b-6568-4b00-9eca-97d0ea0bd5a1/tag/#{@logglyTag}/",
|
206
|
+
body: messageBody.to_json,
|
207
|
+
headers: { 'Content-Type' => 'application/json' } )
|
208
|
+
rescue
|
209
|
+
true
|
210
|
+
end
|
211
|
+
|
212
|
+
else
|
213
|
+
@logger.info "Logging to Loggly disabled"
|
214
|
+
@logger.info messageBody
|
215
|
+
end
|
216
|
+
|
217
|
+
end
|
218
|
+
|
219
|
+
def setupLoggly(tag)
|
220
|
+
@logglyTag = tag
|
221
|
+
end
|
222
|
+
|
174
223
|
end
|
@@ -1,6 +1,12 @@
|
|
1
1
|
require Gem::Specification.find_by_name("navi_client").gem_dir+"/lib/client"
|
2
|
+
require 'concurrent'
|
2
3
|
|
3
4
|
module NaviClient
|
5
|
+
|
6
|
+
##
|
7
|
+
# This class represents the client for cloud version.
|
8
|
+
# In cloud version, all the content will be saved and also being used from the s3
|
9
|
+
#
|
4
10
|
class Cloud
|
5
11
|
include Concurrent::Async
|
6
12
|
include Client
|
@@ -31,6 +37,8 @@ module NaviClient
|
|
31
37
|
# client_type
|
32
38
|
@client_type = "cloud"
|
33
39
|
|
40
|
+
@download_path = config[:s3_download_folder] + '/'
|
41
|
+
|
34
42
|
# set email_address of current_user for s3 folder name
|
35
43
|
@current_user_email = nil
|
36
44
|
end
|
@@ -52,6 +60,9 @@ module NaviClient
|
|
52
60
|
@current_user_email = email
|
53
61
|
end
|
54
62
|
|
63
|
+
##
|
64
|
+
# send bulk request to navi-ai service with list of input files downloaded from imap server.
|
65
|
+
#
|
55
66
|
def send_request(in_filenames = [])
|
56
67
|
unless in_filenames.blank?
|
57
68
|
download_path = config['s3_download_folder'] + "/" + @current_user_email
|
@@ -62,6 +73,9 @@ module NaviClient
|
|
62
73
|
end
|
63
74
|
end
|
64
75
|
|
76
|
+
##
|
77
|
+
# Downloads the email content from imap-server and save it to the download_path
|
78
|
+
#
|
65
79
|
def download(message, custom_uid)
|
66
80
|
download_path = config[:s3_download_folder] + "/" + @current_user_email
|
67
81
|
if ['text/plain', 'text/html'].include? message.mime_type
|
@@ -77,6 +91,10 @@ module NaviClient
|
|
77
91
|
end
|
78
92
|
end
|
79
93
|
|
94
|
+
##
|
95
|
+
# save data to download_path with file named by filename params.
|
96
|
+
# Input is hashmap, and it save the hashmap as yml format.
|
97
|
+
#
|
80
98
|
def save(data={}, filename)
|
81
99
|
download_path = config[:s3_download_folder] + "/" + @current_user_email
|
82
100
|
filepath = download_path + "/" + filename + ".yml"
|
@@ -84,6 +102,8 @@ module NaviClient
|
|
84
102
|
return upload_to_s3(filepath, data.to_yaml)
|
85
103
|
end
|
86
104
|
|
105
|
+
##
|
106
|
+
# Helper function to upload the content to the s3
|
87
107
|
def upload_to_s3(file_path, content)
|
88
108
|
credentials = Aws::Credentials.new(config[:aws_key], config[:aws_secret])
|
89
109
|
s3 = Aws::S3::Client.new(credentials: credentials, region: config[:aws_region])
|
@@ -1,6 +1,11 @@
|
|
1
1
|
require_relative '../client'
|
2
2
|
|
3
3
|
module NaviClient
|
4
|
+
|
5
|
+
##
|
6
|
+
# This class represents the client for local lockbox mode.
|
7
|
+
# In local-lockbox mode, all the content will be saved and also being used from the local file system
|
8
|
+
#
|
4
9
|
class Local
|
5
10
|
include Client
|
6
11
|
CONFIG_PATH = File.join(ENV['HOME'], '/.navi/config.yml')
|
@@ -24,6 +29,8 @@ module NaviClient
|
|
24
29
|
# authentication token received from sso_web used to authenticate the request to database_api
|
25
30
|
@token = nil
|
26
31
|
|
32
|
+
@download_path = config['download_path']
|
33
|
+
|
27
34
|
# client_type
|
28
35
|
@client_type = "local"
|
29
36
|
end
|
@@ -46,8 +53,11 @@ module NaviClient
|
|
46
53
|
@token = "Token: #{token}##{config['username']}"
|
47
54
|
end
|
48
55
|
|
56
|
+
##
|
57
|
+
# Downloads the email content from imap-server and save it to the download_path
|
58
|
+
#
|
49
59
|
def download(message, custom_uid)
|
50
|
-
download_path = config['download_path']
|
60
|
+
download_path = config['download_path'] + config['username'] + '/'
|
51
61
|
if ['text/plain', 'text/html'].include? message.mime_type
|
52
62
|
|
53
63
|
h = Hash.new
|
@@ -62,8 +72,12 @@ module NaviClient
|
|
62
72
|
end
|
63
73
|
end
|
64
74
|
|
75
|
+
##
|
76
|
+
# save data to download_path with file named by filename params.
|
77
|
+
# Input is hashmap, and it save the hashmap as yml format.
|
78
|
+
#
|
65
79
|
def save(data={}, filename)
|
66
|
-
download_path = config['download_path']
|
80
|
+
download_path = config['download_path'] + config['username'] + '/'
|
67
81
|
filepath = download_path + filename + ".yml"
|
68
82
|
|
69
83
|
mkdir_if_not_exist(filepath)
|
@@ -72,10 +86,13 @@ module NaviClient
|
|
72
86
|
return filepath
|
73
87
|
end
|
74
88
|
|
89
|
+
##
|
90
|
+
# send bulk request to navi-ai service with list of input files downloaded from imap server.
|
91
|
+
#
|
75
92
|
def send_request(in_filenames = [])
|
76
93
|
unless in_filenames.empty?
|
77
94
|
download_path = config['download_path']
|
78
|
-
filename = download_path + "inputs/" + (Time.now.to_f * 1000).to_s
|
95
|
+
filename = download_path + config['username'] + "/inputs/" + (Time.now.to_f * 1000).to_s
|
79
96
|
|
80
97
|
mkdir_if_not_exist(filename)
|
81
98
|
|
@@ -122,7 +139,7 @@ module NaviClient
|
|
122
139
|
# We're out, which means there are some emails ready for us.
|
123
140
|
# Go do a search for UNSEEN and fetch them.
|
124
141
|
filenames = []
|
125
|
-
retrieve_emails(imap, search_condition, folder) { |mail| filenames << process_email(mail)}
|
142
|
+
retrieve_emails(imap, search_condition, folder) { |mail, i, isLast, id| filenames << process_email(mail, id)}
|
126
143
|
self.send_request(filenames)
|
127
144
|
|
128
145
|
HTTPService::NaviAI.generate_csv(username)
|
data/lib/navi_client.rb
CHANGED
data/lib/navi_client/version.rb
CHANGED
data/navi_client.gemspec
CHANGED
metadata
CHANGED
@@ -1,87 +1,101 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: navi_client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.3.
|
4
|
+
version: 1.3.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sujit
|
8
8
|
- Surya
|
9
9
|
- Saroj
|
10
|
-
autorequire:
|
10
|
+
autorequire:
|
11
11
|
bindir: exe
|
12
12
|
cert_chain: []
|
13
|
-
date: 2018-
|
13
|
+
date: 2018-03-06 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
|
+
name: bundler
|
16
17
|
requirement: !ruby/object:Gem::Requirement
|
17
18
|
requirements:
|
18
19
|
- - "~>"
|
19
20
|
- !ruby/object:Gem::Version
|
20
21
|
version: '1.11'
|
21
|
-
name: bundler
|
22
|
-
prerelease: false
|
23
22
|
type: :development
|
23
|
+
prerelease: false
|
24
24
|
version_requirements: !ruby/object:Gem::Requirement
|
25
25
|
requirements:
|
26
26
|
- - "~>"
|
27
27
|
- !ruby/object:Gem::Version
|
28
28
|
version: '1.11'
|
29
29
|
- !ruby/object:Gem::Dependency
|
30
|
+
name: rake
|
30
31
|
requirement: !ruby/object:Gem::Requirement
|
31
32
|
requirements:
|
32
33
|
- - "~>"
|
33
34
|
- !ruby/object:Gem::Version
|
34
35
|
version: '10.0'
|
35
|
-
name: rake
|
36
|
-
prerelease: false
|
37
36
|
type: :development
|
37
|
+
prerelease: false
|
38
38
|
version_requirements: !ruby/object:Gem::Requirement
|
39
39
|
requirements:
|
40
40
|
- - "~>"
|
41
41
|
- !ruby/object:Gem::Version
|
42
42
|
version: '10.0'
|
43
43
|
- !ruby/object:Gem::Dependency
|
44
|
+
name: rspec
|
44
45
|
requirement: !ruby/object:Gem::Requirement
|
45
46
|
requirements:
|
46
47
|
- - "~>"
|
47
48
|
- !ruby/object:Gem::Version
|
48
49
|
version: '3.0'
|
49
|
-
name: rspec
|
50
|
-
prerelease: false
|
51
50
|
type: :development
|
51
|
+
prerelease: false
|
52
52
|
version_requirements: !ruby/object:Gem::Requirement
|
53
53
|
requirements:
|
54
54
|
- - "~>"
|
55
55
|
- !ruby/object:Gem::Version
|
56
56
|
version: '3.0'
|
57
57
|
- !ruby/object:Gem::Dependency
|
58
|
+
name: httparty
|
58
59
|
requirement: !ruby/object:Gem::Requirement
|
59
60
|
requirements:
|
60
61
|
- - ">="
|
61
62
|
- !ruby/object:Gem::Version
|
62
63
|
version: '0'
|
63
|
-
name: httparty
|
64
|
-
prerelease: false
|
65
64
|
type: :runtime
|
65
|
+
prerelease: false
|
66
66
|
version_requirements: !ruby/object:Gem::Requirement
|
67
67
|
requirements:
|
68
68
|
- - ">="
|
69
69
|
- !ruby/object:Gem::Version
|
70
70
|
version: '0'
|
71
71
|
- !ruby/object:Gem::Dependency
|
72
|
+
name: mail
|
72
73
|
requirement: !ruby/object:Gem::Requirement
|
73
74
|
requirements:
|
74
75
|
- - ">="
|
75
76
|
- !ruby/object:Gem::Version
|
76
77
|
version: '0'
|
77
|
-
name: mail
|
78
|
-
prerelease: false
|
79
78
|
type: :runtime
|
79
|
+
prerelease: false
|
80
80
|
version_requirements: !ruby/object:Gem::Requirement
|
81
81
|
requirements:
|
82
82
|
- - ">="
|
83
83
|
- !ruby/object:Gem::Version
|
84
84
|
version: '0'
|
85
|
+
- !ruby/object:Gem::Dependency
|
86
|
+
name: concurrent-ruby
|
87
|
+
requirement: !ruby/object:Gem::Requirement
|
88
|
+
requirements:
|
89
|
+
- - "~>"
|
90
|
+
- !ruby/object:Gem::Version
|
91
|
+
version: 1.0.5
|
92
|
+
type: :runtime
|
93
|
+
prerelease: false
|
94
|
+
version_requirements: !ruby/object:Gem::Requirement
|
95
|
+
requirements:
|
96
|
+
- - "~>"
|
97
|
+
- !ruby/object:Gem::Version
|
98
|
+
version: 1.0.5
|
85
99
|
description: " Write a longer description or delete this line."
|
86
100
|
email:
|
87
101
|
- sujit@whitehatengineering.com
|
@@ -113,7 +127,7 @@ homepage: http://navihq.com
|
|
113
127
|
licenses:
|
114
128
|
- MIT
|
115
129
|
metadata: {}
|
116
|
-
post_install_message:
|
130
|
+
post_install_message:
|
117
131
|
rdoc_options: []
|
118
132
|
require_paths:
|
119
133
|
- lib
|
@@ -128,9 +142,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
128
142
|
- !ruby/object:Gem::Version
|
129
143
|
version: '0'
|
130
144
|
requirements: []
|
131
|
-
rubyforge_project:
|
132
|
-
rubygems_version: 2.
|
133
|
-
signing_key:
|
145
|
+
rubyforge_project:
|
146
|
+
rubygems_version: 2.7.4
|
147
|
+
signing_key:
|
134
148
|
specification_version: 4
|
135
149
|
summary: Write a short summary, because Rubygems requires one.
|
136
150
|
test_files: []
|