ishapi 0.1.8.302 → 0.1.8.304
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/app/jobs/ishapi/email_message_intake_job.rb +161 -213
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c2aa5889cc1b26469a45c1177f999974ab3c8d3dead962628024eaf53b22ffdf
|
4
|
+
data.tar.gz: ac848af52069580c948d4f9f25b3bb909b9114647bcdb2dd967dd963dc548513
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c3702b34c60967c73931108100513252416ba21dec37e7365276ec2c4eee3b97135d275f19c50b80b0a9611c3cbfdf5eb84d84ac75d30a2ca5842acb948be1a9
|
7
|
+
data.tar.gz: e540871e032d8f92633ad726ba51b6e17b14430688ea7af76637f880ec54676fbbb55c4fd0ae6ef8a31dccb883f460143844a5fdefac118586110ce19a0227dd
|
@@ -9,243 +9,191 @@ class Ishapi::EmailMessageIntakeJob < Ishapi::ApplicationJob
|
|
9
9
|
|
10
10
|
queue_as :default
|
11
11
|
|
12
|
-
|
13
|
-
## Content dispositions:
|
14
|
-
# "inline; creation-date=\"Tue, 11 Apr 2023 19:39:42 GMT\"; filename=image005.png; modification-date=\"Tue, 11 Apr 2023 19:47:53 GMT\"; size=14916",
|
15
|
-
#
|
16
|
-
## Content Types:
|
17
|
-
# "application/pdf; name=\"Securities Forward Agreement -- HaulHub Inc -- Victor Pudeyev -- 2021-10-26.docx.pdf\""
|
18
|
-
# "image/jpeg; name=TX_DL_2.jpg"
|
19
|
-
# "image/png; name=image005.png"
|
20
|
-
# "multipart/alternative; boundary=_000_BL0PR10MB2913C560ADE059F0AB3A6D11829A9BL0PR10MB2913namp_",
|
21
|
-
# "text/html; charset=utf-8"
|
22
|
-
# "text/plain; charset=UTF-8"
|
23
|
-
# "text/calendar; charset=utf-8; method=REQUEST"
|
24
|
-
def churn_subpart message, part
|
25
|
-
if part.content_disposition&.include?('attachment')
|
26
|
-
## @TODO: attachments !
|
27
|
-
;
|
28
|
-
else
|
29
|
-
if part.content_type.include?("multipart/related") ||
|
30
|
-
part.content_type.include?("multipart/alternative")
|
31
|
-
|
32
|
-
part.parts.each do |subpart|
|
33
|
-
churn_subpart( message, subpart )
|
34
|
-
end
|
35
|
-
else
|
36
|
-
attachment = Office::EmailAttachment.new({
|
37
|
-
content: part.decoded,
|
38
|
-
content_type: part.content_type,
|
39
|
-
email_message: message,
|
40
|
-
})
|
41
|
-
attachment.save
|
12
|
+
=begin
|
42
13
|
|
43
|
-
|
44
|
-
|
14
|
+
object_key = 'n0v5mg6q1t4fjjnfh8vj8a96t85rp9la2ud0gdg1'
|
15
|
+
MsgStub.where({ object_key: object_key }).delete
|
45
16
|
|
46
|
-
|
47
|
-
|
17
|
+
stub = MsgStub.create!({ object_key: object_key })
|
18
|
+
id = stub.id
|
48
19
|
|
49
|
-
|
50
|
-
;
|
51
|
-
elsif part.content_type.include?("application/pdf")
|
52
|
-
;
|
53
|
-
elsif part.content_type.include?("image/jpeg")
|
54
|
-
;
|
55
|
-
elsif part.content_type.include?("image/png")
|
56
|
-
;
|
20
|
+
Ishapi::EmailMessageIntakeJob.perform_now( stub.id.to_s )
|
57
21
|
|
58
|
-
|
59
|
-
|
22
|
+
=end
|
23
|
+
def perform id
|
24
|
+
stub = ::Office::EmailMessageStub.find id
|
25
|
+
if !Rails.env.test?
|
26
|
+
puts "Performing EmailMessageIntakeJob for object_key #{stub.object_key}"
|
27
|
+
end
|
28
|
+
if stub.state != ::Office::EmailMessageStub::STATE_PENDING
|
29
|
+
raise "This stub has already been processed: #{stub.id.to_s}."
|
30
|
+
return
|
31
|
+
end
|
60
32
|
|
61
|
-
|
33
|
+
client = Aws::S3::Client.new({
|
34
|
+
region: ::S3_CREDENTIALS[:region_ses],
|
35
|
+
access_key_id: ::S3_CREDENTIALS[:access_key_id_ses],
|
36
|
+
secret_access_key: ::S3_CREDENTIALS[:secret_access_key_ses],
|
37
|
+
})
|
38
|
+
|
39
|
+
_mail = client.get_object( bucket: ::S3_CREDENTIALS[:bucket_ses], key: stub.object_key ).body.read
|
40
|
+
the_mail = Mail.new(_mail)
|
41
|
+
message_id = the_mail.header['message-id'].decoded
|
42
|
+
in_reply_to_id = the_mail.header['in-reply-to']&.to_s
|
43
|
+
email_inbox_tag_id = WpTag.emailtag(WpTag::INBOX).id
|
44
|
+
|
45
|
+
if !the_mail.to
|
46
|
+
the_mail.to = [ 'NO-RECIPIENT' ]
|
62
47
|
end
|
63
|
-
end
|
64
|
-
end
|
65
48
|
|
66
|
-
## From: https://stackoverflow.com/questions/24672834/how-do-i-remove-emoji-from-string/24673322
|
67
|
-
def strip_emoji(text)
|
68
|
-
text = text.force_encoding('utf-8').encode
|
69
|
-
clean = ""
|
70
49
|
|
71
|
-
|
72
|
-
|
73
|
-
clean = text.gsub regex, ""
|
50
|
+
subject = ::Msg.strip_emoji the_mail.subject
|
51
|
+
subject ||= '(wco no subject)'
|
74
52
|
|
75
|
-
|
76
|
-
|
77
|
-
|
53
|
+
@message = ::Office::EmailMessage.where( message_id: message_id ).first
|
54
|
+
@message ||= ::Office::EmailMessage.create({
|
55
|
+
raw: _mail,
|
78
56
|
|
79
|
-
|
80
|
-
|
81
|
-
clean = clean.gsub regex, ""
|
57
|
+
message_id: message_id,
|
58
|
+
in_reply_to_id: in_reply_to_id,
|
82
59
|
|
83
|
-
|
84
|
-
|
85
|
-
clean = clean.gsub regex, ""
|
86
|
-
end
|
60
|
+
object_key: stub.object_key,
|
61
|
+
# object_path: stub.object_path,
|
87
62
|
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
})
|
102
|
-
|
103
|
-
_mail = client.get_object( bucket: ::S3_CREDENTIALS[:bucket_ses], key: stub.object_key ).body.read
|
104
|
-
the_mail = Mail.new(_mail)
|
105
|
-
message_id = the_mail.header['message-id'].decoded
|
106
|
-
in_reply_to_id = the_mail.header['in-reply-to']&.to_s
|
107
|
-
email_inbox_tag_id = WpTag.emailtag(WpTag::INBOX).id
|
108
|
-
|
109
|
-
if !the_mail.to
|
110
|
-
the_mail.to = [ 'NO-RECIPIENT' ]
|
111
|
-
end
|
112
|
-
|
113
|
-
|
114
|
-
subject = strip_emoji the_mail.subject
|
115
|
-
subject ||= '(wco no subject)'
|
116
|
-
|
117
|
-
@message = ::Office::EmailMessage.where( message_id: message_id ).first
|
118
|
-
@message ||= ::Office::EmailMessage.create({
|
119
|
-
raw: _mail,
|
120
|
-
|
121
|
-
message_id: message_id,
|
122
|
-
in_reply_to_id: in_reply_to_id,
|
123
|
-
|
124
|
-
object_key: stub.object_key,
|
125
|
-
# object_path: stub.object_path,
|
126
|
-
|
127
|
-
subject: subject,
|
128
|
-
date: the_mail.date,
|
129
|
-
|
130
|
-
from: the_mail.from ? the_mail.from[0] : "nobody@unknown-doma.in",
|
131
|
-
froms: the_mail.from,
|
132
|
-
|
133
|
-
to: the_mail.to ? the_mail.to[0] : nil,
|
134
|
-
tos: the_mail.to,
|
135
|
-
|
136
|
-
cc: the_mail.cc ? the_mail.cc[0] : nil,
|
137
|
-
ccs: the_mail.cc,
|
138
|
-
|
139
|
-
# bccs: the_mail.bcc,
|
140
|
-
})
|
141
|
-
if !@message.persisted?
|
142
|
-
puts! @message.errors.full_messages, "Could not create email_message"
|
143
|
-
end
|
144
|
-
if the_mail.body.preamble.present?
|
145
|
-
@message.preamble = the_mail.body.preamble
|
146
|
-
end
|
147
|
-
if the_mail.body.epilogue.present?
|
148
|
-
@message.epilogue = the_mail.body.epilogue
|
149
|
-
end
|
150
|
-
|
151
|
-
## Parts
|
152
|
-
the_mail.parts.each do |part|
|
153
|
-
churn_subpart( @message, part )
|
154
|
-
end
|
155
|
-
|
156
|
-
if the_mail.parts.length == 0
|
157
|
-
body = the_mail.body.decoded.encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
|
158
|
-
if the_mail.content_type&.include?('text/html')
|
159
|
-
@message.part_html = body
|
160
|
-
elsif the_mail.content_type&.include?('text/plain')
|
161
|
-
@message.part_txt = body
|
162
|
-
elsif the_mail.content_type.blank?
|
163
|
-
@message.part_txt = body
|
164
|
-
else
|
165
|
-
throw "mail body of unknown type: #{the_mail.content_type}"
|
166
|
-
end
|
167
|
-
end
|
168
|
-
|
169
|
-
## Attachments
|
170
|
-
the_mail.attachments.each do |att|
|
171
|
-
photo = Photo.new({
|
172
|
-
content_type: att.content_type.split(';')[0],
|
173
|
-
original_filename: att.content_type_parameters[:name],
|
174
|
-
image_data: att.body.encoded,
|
175
|
-
email_message_id: @message.id,
|
63
|
+
subject: subject,
|
64
|
+
date: the_mail.date,
|
65
|
+
|
66
|
+
from: the_mail.from ? the_mail.from[0] : "nobody@unknown-doma.in",
|
67
|
+
froms: the_mail.from,
|
68
|
+
|
69
|
+
to: the_mail.to ? the_mail.to[0] : nil,
|
70
|
+
tos: the_mail.to,
|
71
|
+
|
72
|
+
cc: the_mail.cc ? the_mail.cc[0] : nil,
|
73
|
+
ccs: the_mail.cc,
|
74
|
+
|
75
|
+
# bccs: the_mail.bcc,
|
176
76
|
})
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
77
|
+
if !@message.persisted?
|
78
|
+
puts! @message.errors.full_messages, "Could not create email_message"
|
79
|
+
end
|
80
|
+
if the_mail.body.preamble.present?
|
81
|
+
@message.preamble = the_mail.body.preamble
|
82
|
+
end
|
83
|
+
if the_mail.body.epilogue.present?
|
84
|
+
@message.epilogue = the_mail.body.epilogue
|
85
|
+
end
|
86
|
+
|
87
|
+
## Parts
|
88
|
+
the_mail.parts.each do |part|
|
89
|
+
@message.churn_subpart( part )
|
90
|
+
end
|
91
|
+
|
92
|
+
if the_mail.parts.length == 0
|
93
|
+
body = the_mail.body.decoded.encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
|
94
|
+
if the_mail.content_type&.include?('text/html')
|
95
|
+
@message.part_html = body
|
96
|
+
elsif the_mail.content_type&.include?('text/plain')
|
97
|
+
@message.part_txt = body
|
98
|
+
elsif the_mail.content_type.blank?
|
99
|
+
@message.part_txt = body
|
100
|
+
else
|
101
|
+
@message.logs.push "mail body of unknown type: #{the_mail.content_type}"
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
## Attachments
|
106
|
+
the_mail.attachments.each do |att|
|
107
|
+
content_type = att.content_type.split(';')[0]
|
108
|
+
if content_type.include? 'image'
|
109
|
+
photo = Photo.new({
|
110
|
+
content_type: content_type,
|
111
|
+
original_filename: att.content_type_parameters[:name],
|
112
|
+
image_data: att.body.encoded,
|
113
|
+
email_message_id: @message.id,
|
114
|
+
})
|
115
|
+
photo.decode_base64_image
|
116
|
+
photo.save
|
117
|
+
elsif att.filename
|
118
|
+
attachment = Office::EmailAttachment.new({
|
119
|
+
content: att.body.decoded,
|
120
|
+
content_type: att.content_type,
|
121
|
+
email_message: @message,
|
122
|
+
filename: att.filename,
|
123
|
+
})
|
124
|
+
attachment.save
|
125
|
+
else
|
126
|
+
@message.logs.push "Could not save an attachment!"
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
## Leadset, Lead
|
131
|
+
domain = @message.from.split('@')[1] rescue 'unknown.domain'
|
187
132
|
leadset = Leadset.find_or_create_by( company_url: domain )
|
188
|
-
Lead.find_or_create_by( email:
|
189
|
-
|
133
|
+
lead = Lead.find_or_create_by( email: @message.from, m3_leadset_id: leadset.id )
|
134
|
+
the_mail.cc&.each do |cc|
|
135
|
+
domain = cc.split('@')[1] rescue 'unknown.domain'
|
136
|
+
leadset = Leadset.find_or_create_by( company_url: domain )
|
137
|
+
Lead.find_or_create_by( email: cc, m3_leadset_id: leadset.id )
|
138
|
+
end
|
190
139
|
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
140
|
+
## Conversation
|
141
|
+
if in_reply_to_id
|
142
|
+
in_reply_to_msg = ::Office::EmailMessage.where({ message_id: in_reply_to_id }).first
|
143
|
+
if !in_reply_to_msg
|
144
|
+
conv = ::Office::EmailConversation.find_or_create_by({
|
145
|
+
subject: @message.subject,
|
146
|
+
})
|
147
|
+
in_reply_to_msg = ::Office::EmailMessage.find_or_create_by({
|
148
|
+
message_id: in_reply_to_id,
|
149
|
+
email_conversation_id: conv.id,
|
150
|
+
})
|
151
|
+
end
|
152
|
+
conv = in_reply_to_msg.email_conversation
|
153
|
+
else
|
195
154
|
conv = ::Office::EmailConversation.find_or_create_by({
|
196
155
|
subject: @message.subject,
|
197
156
|
})
|
198
|
-
in_reply_to_msg = ::Office::EmailMessage.find_or_create_by({
|
199
|
-
message_id: in_reply_to_id,
|
200
|
-
email_conversation_id: conv.id,
|
201
|
-
})
|
202
157
|
end
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
158
|
+
@message.update_attributes({ email_conversation_id: conv.id })
|
159
|
+
conv.update_attributes({
|
160
|
+
state: Conv::STATE_UNREAD,
|
161
|
+
latest_at: the_mail.date || Time.now.to_datetime,
|
162
|
+
from_emails: ( conv.from_emails + the_mail.from ).uniq,
|
163
|
+
})
|
164
|
+
conv.add_tag( ::WpTag::INBOX )
|
165
|
+
conv_lead_tie = Office::EmailConversationLead.find_or_create_by({
|
166
|
+
lead_id: lead.id,
|
167
|
+
email_conversation_id: conv.id,
|
207
168
|
})
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
if ( filter.from_regex.blank? || @message.from.match( filter.from_regex ) ) &&
|
226
|
-
( filter.from_exact.blank? || @message.from.downcase.include?( filter.from_exact&.downcase ) ) &&
|
227
|
-
( filter.body_exact.blank? || @message.part_html&.include?( filter.body_exact ) ) &&
|
228
|
-
( filter.subject_regex.blank? || @message.subject.match( filter.subject_regex ) ) &&
|
229
|
-
( filter.subject_exact.blank? || @message.subject.downcase.include?( filter.subject_exact&.downcase ) )
|
230
|
-
|
231
|
-
# || MiaTagger.analyze( @message.part_html, :is_spammy_recruite ).score > .5
|
232
|
-
|
233
|
-
puts! "applying filter #{filter} to conv #{conv}" if DEBUG
|
234
|
-
|
235
|
-
@message.apply_filter( filter )
|
169
|
+
|
170
|
+
|
171
|
+
## Actions & Filters
|
172
|
+
email_filters = Office::EmailFilter.active
|
173
|
+
email_filters.each do |filter|
|
174
|
+
if ( filter.from_regex.blank? || @message.from.match( filter.from_regex ) ) &&
|
175
|
+
( filter.from_exact.blank? || @message.from.downcase.include?( filter.from_exact&.downcase ) ) &&
|
176
|
+
( filter.body_exact.blank? || @message.part_html&.include?( filter.body_exact ) ) &&
|
177
|
+
( filter.subject_regex.blank? || @message.subject.match( filter.subject_regex ) ) &&
|
178
|
+
( filter.subject_exact.blank? || @message.subject.downcase.include?( filter.subject_exact&.downcase ) )
|
179
|
+
|
180
|
+
# || MiaTagger.analyze( @message.part_html, :is_spammy_recruite ).score > .5
|
181
|
+
|
182
|
+
puts! "applying filter #{filter} to conv #{conv}" if DEBUG
|
183
|
+
|
184
|
+
@message.apply_filter( filter )
|
185
|
+
end
|
236
186
|
end
|
237
|
-
end
|
238
187
|
|
239
|
-
|
188
|
+
stub.update_attributes({ state: ::Office::EmailMessageStub::STATE_PROCESSED })
|
240
189
|
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
190
|
+
## Notification
|
191
|
+
conv = Conv.find( conv.id )
|
192
|
+
if conv.in_emailtag? WpTag::INBOX
|
193
|
+
out = ::Ishapi::ApplicationMailer.forwarder_notify( @message.id.to_s )
|
194
|
+
Rails.env.production? ? out.deliver_later : out.deliver_now
|
195
|
+
end
|
247
196
|
|
248
197
|
end
|
249
|
-
|
250
198
|
end
|
251
199
|
EIJ = Ishapi::EmailMessageIntakeJob
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ishapi
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.8.
|
4
|
+
version: 0.1.8.304
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- piousbox
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-11-
|
11
|
+
date: 2023-11-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|