ishapi 0.1.8.302 → 0.1.8.303

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 57efdeeb134f2713ceff675afdcd38227765df6cd5769ef9f85423c79d46999e
-  data.tar.gz: a7ad58a601430d8172229fd66dc9b612a8ca432b7ea1602741c8fd6a54f3f561
+  metadata.gz: deb042effc76a3bac981e7d95c5ccbac9c1fe067b5afb6799f273a2fb061ba67
+  data.tar.gz: a497b8b725895391c965abe2680adb51d739958ca36172615263aa3fd6b83703
 SHA512:
-  metadata.gz: 16c58f71d63bf5b156fe7800f66505b0ec3c7b607d4900b89dcebbd99a7e4b98218c12af11c63b5df9320d546ad212efe1d7a302446a7bbb3763a76a42e7e0f7
-  data.tar.gz: c68af62bda4af804a79bd8e2a74349507a40579a2c26784496f6c65ffd24c0b869dc27c2b46bb7c79285c17c666a03cb0f3cc011fde56cb970544b99b7f2e95b
+  metadata.gz: 206d685ea05a79c226a49d888e1ba2ced49e5ff32343999fe1a00e42c65eb8f139e17cc1dcb2bd6d0e7acbb83c257ef55205d75f318631c353a3aef22b6db143
+  data.tar.gz: ceebf0b9f54feb97eeac59cc655d65c12fc10957431d4821bf472e18822d97d80be31b0407f914a2a94e63119c62a5cf8a743e9f96556f088c481e90fbad3eff

data/app/jobs/ishapi/email_message_intake_job.rb

@@ -9,243 +9,183 @@ class Ishapi::EmailMessageIntakeJob < Ishapi::ApplicationJob
 
   queue_as :default
 
-  ## For recursive parts of type `related`.
-  ## Content dispositions:
-  # "inline; creation-date=\"Tue, 11 Apr 2023 19:39:42 GMT\"; filename=image005.png; modification-date=\"Tue, 11 Apr 2023 19:47:53 GMT\"; size=14916",
-  #
-  ## Content Types:
-  # "application/pdf; name=\"Securities Forward Agreement -- HaulHub Inc -- Victor Pudeyev -- 2021-10-26.docx.pdf\""
-  # "image/jpeg; name=TX_DL_2.jpg"
-  # "image/png; name=image005.png"
-  # "multipart/alternative; boundary=_000_BL0PR10MB2913C560ADE059F0AB3A6D11829A9BL0PR10MB2913namp_",
-  # "text/html; charset=utf-8"
-  # "text/plain; charset=UTF-8"
-  # "text/calendar; charset=utf-8; method=REQUEST"
-  def churn_subpart message, part
-    if part.content_disposition&.include?('attachment')
-      ## @TODO: attachments !
-      ;
-    else
-      if part.content_type.include?("multipart/related") ||
-        part.content_type.include?("multipart/alternative")
-
-        part.parts.each do |subpart|
-          churn_subpart( message, subpart )
-        end
-      else
-        attachment = Office::EmailAttachment.new({
-          content:       part.decoded,
-          content_type:  part.content_type,
-          email_message: message,
-        })
-        attachment.save
+=begin
+  object_key = 'bg57j6u0j38b5ts86635fkqtjlucn5tvrm2hea81'
+  MsgStub.where({ object_key: object_key }).delete
 
-        if part.content_type.include?('text/html')
-          message.part_html = part.decoded
+  stub = MsgStub.create({ object_key: object_key })
+  id = stub.id
+=end
+  def perform id
+      stub = ::Office::EmailMessageStub.find id
+      if !Rails.env.test?
+        puts "Performing EmailMessageIntakeJob for object_key #{stub.object_key}"
+      end
+      if stub.state != ::Office::EmailMessageStub::STATE_PENDING
+        raise "This stub has already been processed: #{stub.id.to_s}."
+        return
+      end
+      client = Aws::S3::Client.new({
+        region:            ::S3_CREDENTIALS[:region_ses],
+        access_key_id:     ::S3_CREDENTIALS[:access_key_id_ses],
+        secret_access_key: ::S3_CREDENTIALS[:secret_access_key_ses],
+      })
 
-        elsif part.content_type.include?("text/plain")
-          message.part_txt = part.decoded
+      _mail              = client.get_object( bucket: ::S3_CREDENTIALS[:bucket_ses], key: stub.object_key ).body.read
+      the_mail           = Mail.new(_mail)
+      message_id         = the_mail.header['message-id'].decoded
+      in_reply_to_id     = the_mail.header['in-reply-to']&.to_s
+      email_inbox_tag_id = WpTag.emailtag(WpTag::INBOX).id
 
-        elsif part.content_type.include?("text/calendar")
-          ;
-        elsif part.content_type.include?("application/pdf")
-          ;
-        elsif part.content_type.include?("image/jpeg")
-          ;
-        elsif part.content_type.include?("image/png")
-          ;
+      if !the_mail.to
+        the_mail.to = [ 'NO-RECIPIENT' ]
+      end
 
-        else
-          puts! part.content_type, '444 No action for a part with this content_type'
 
-        end
-      end
-    end
-  end
+      subject   = ::Msg.strip_emoji the_mail.subject
+      subject ||= '(wco no subject)'
 
-  ## From: https://stackoverflow.com/questions/24672834/how-do-i-remove-emoji-from-string/24673322
-  def strip_emoji(text)
-    text = text.force_encoding('utf-8').encode
-    clean = ""
+      @message   = ::Office::EmailMessage.where( message_id: message_id ).first
+      @message ||= ::Office::EmailMessage.create({
+        raw: _mail,
 
-    # symbols & pics
-    regex = /[\u{1f300}-\u{1f5ff}]/
-    clean = text.gsub regex, ""
+        message_id:     message_id,
+        in_reply_to_id: in_reply_to_id,
 
-    # enclosed chars
-    regex = /[\u{2500}-\u{2BEF}]/ # I changed this to exclude chinese char
-    clean = clean.gsub regex, ""
+        object_key:  stub.object_key,
+        # object_path: stub.object_path,
 
-    # emoticons
-    regex = /[\u{1f600}-\u{1f64f}]/
-    clean = clean.gsub regex, ""
+        subject: subject,
+        date:    the_mail.date,
 
-    #dingbats
-    regex = /[\u{2702}-\u{27b0}]/
-    clean = clean.gsub regex, ""
-  end
+        from:  the_mail.from ? the_mail.from[0] : "nobody@unknown-doma.in",
+        froms: the_mail.from,
 
-  def perform id
-    stub = ::Office::EmailMessageStub.find id
-    if !Rails.env.test?
-      puts "Performing EmailMessageIntakeJob for object_key #{stub.object_key}"
-    end
-    if stub.state != ::Office::EmailMessageStub::STATE_PENDING
-      raise "This stub has already been processed: #{stub.id.to_s}."
-      return
-    end
-    client = Aws::S3::Client.new({
-      region:            ::S3_CREDENTIALS[:region_ses],
-      access_key_id:     ::S3_CREDENTIALS[:access_key_id_ses],
-      secret_access_key: ::S3_CREDENTIALS[:secret_access_key_ses],
-    })
-
-    _mail              = client.get_object( bucket: ::S3_CREDENTIALS[:bucket_ses], key: stub.object_key ).body.read
-    the_mail           = Mail.new(_mail)
-    message_id         = the_mail.header['message-id'].decoded
-    in_reply_to_id     = the_mail.header['in-reply-to']&.to_s
-    email_inbox_tag_id = WpTag.emailtag(WpTag::INBOX).id
-
-    if !the_mail.to
-      the_mail.to = [ 'NO-RECIPIENT' ]
-    end
-
-
-    subject   = strip_emoji the_mail.subject
-    subject ||= '(wco no subject)'
-
-    @message   = ::Office::EmailMessage.where( message_id: message_id ).first
-    @message ||= ::Office::EmailMessage.create({
-      raw: _mail,
-
-      message_id:     message_id,
-      in_reply_to_id: in_reply_to_id,
-
-      object_key:  stub.object_key,
-      # object_path: stub.object_path,
-
-      subject: subject,
-      date:    the_mail.date,
-
-      from:  the_mail.from ? the_mail.from[0] : "nobody@unknown-doma.in",
-      froms: the_mail.from,
-
-      to:  the_mail.to ? the_mail.to[0] : nil,
-      tos: the_mail.to,
-
-      cc:  the_mail.cc ? the_mail.cc[0] : nil,
-      ccs:  the_mail.cc,
-
-      # bccs: the_mail.bcc,
-    })
-    if !@message.persisted?
-      puts! @message.errors.full_messages, "Could not create email_message"
-    end
-    if the_mail.body.preamble.present?
-      @message.preamble = the_mail.body.preamble
-    end
-    if the_mail.body.epilogue.present?
-      @message.epilogue = the_mail.body.epilogue
-    end
-
-    ## Parts
-    the_mail.parts.each do |part|
-      churn_subpart( @message, part )
-    end
-
-    if the_mail.parts.length == 0
-      body = the_mail.body.decoded.encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
-      if the_mail.content_type&.include?('text/html')
-        @message.part_html = body
-      elsif the_mail.content_type&.include?('text/plain')
-        @message.part_txt = body
-      elsif the_mail.content_type.blank?
-        @message.part_txt = body
-      else
-        throw "mail body of unknown type: #{the_mail.content_type}"
-      end
-    end
-
-    ## Attachments
-    the_mail.attachments.each do |att|
-      photo = Photo.new({
-        content_type:      att.content_type.split(';')[0],
-        original_filename: att.content_type_parameters[:name],
-        image_data:        att.body.encoded,
-        email_message_id: @message.id,
+        to:  the_mail.to ? the_mail.to[0] : nil,
+        tos: the_mail.to,
+
+        cc:  the_mail.cc ? the_mail.cc[0] : nil,
+        ccs:  the_mail.cc,
+
+        # bccs: the_mail.bcc,
       })
-      photo.decode_base64_image
-      photo.save
-    end
-
-    ## Leadset, Lead
-    domain  = @message.from.split('@')[1] rescue 'unknown.domain'
-    leadset = Leadset.find_or_create_by( company_url: domain )
-    lead    = Lead.find_or_create_by( email: @message.from, m3_leadset_id: leadset.id )
-    the_mail.cc&.each do |cc|
-      domain  = cc.split('@')[1] rescue 'unknown.domain'
+      if !@message.persisted?
+        puts! @message.errors.full_messages, "Could not create email_message"
+      end
+      if the_mail.body.preamble.present?
+        @message.preamble = the_mail.body.preamble
+      end
+      if the_mail.body.epilogue.present?
+        @message.epilogue = the_mail.body.epilogue
+      end
+
+      ## Parts
+      the_mail.parts.each do |part|
+        @message.churn_subpart( part )
+      end
+
+      if the_mail.parts.length == 0
+        body = the_mail.body.decoded.encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
+        if the_mail.content_type&.include?('text/html')
+          @message.part_html = body
+        elsif the_mail.content_type&.include?('text/plain')
+          @message.part_txt = body
+        elsif the_mail.content_type.blank?
+          @message.part_txt = body
+        else
+          throw "mail body of unknown type: #{the_mail.content_type}"
+        end
+      end
+
+      ## Attachments
+      the_mail.attachments.each do |att|
+        photo = Photo.new({
+          content_type:      att.content_type.split(';')[0],
+          original_filename: att.content_type_parameters[:name],
+          image_data:        att.body.encoded,
+          email_message_id: @message.id,
+        })
+        photo.decode_base64_image
+        if photo.save
+          ;
+        else
+          attachment = Office::EmailAttachment.new({
+            content:       att.decoded,
+            content_type:  att.content_type,
+            email_message: @message,
+            filename:      att.filename,
+          })
+          attachment.save
+        end
+      end
+
+      ## Leadset, Lead
+      domain  = @message.from.split('@')[1] rescue 'unknown.domain'
       leadset = Leadset.find_or_create_by( company_url: domain )
-      Lead.find_or_create_by( email: cc, m3_leadset_id: leadset.id )
-    end
+      lead    = Lead.find_or_create_by( email: @message.from, m3_leadset_id: leadset.id )
+      the_mail.cc&.each do |cc|
+        domain  = cc.split('@')[1] rescue 'unknown.domain'
+        leadset = Leadset.find_or_create_by( company_url: domain )
+        Lead.find_or_create_by( email: cc, m3_leadset_id: leadset.id )
+      end
 
-    ## Conversation
-    if in_reply_to_id
-      in_reply_to_msg = ::Office::EmailMessage.where({ message_id: in_reply_to_id }).first
-      if !in_reply_to_msg
+      ## Conversation
+      if in_reply_to_id
+        in_reply_to_msg = ::Office::EmailMessage.where({ message_id: in_reply_to_id }).first
+        if !in_reply_to_msg
+          conv = ::Office::EmailConversation.find_or_create_by({
+            subject: @message.subject,
+          })
+          in_reply_to_msg = ::Office::EmailMessage.find_or_create_by({
+            message_id: in_reply_to_id,
+            email_conversation_id: conv.id,
+          })
+        end
+        conv = in_reply_to_msg.email_conversation
+      else
         conv = ::Office::EmailConversation.find_or_create_by({
           subject: @message.subject,
         })
-        in_reply_to_msg = ::Office::EmailMessage.find_or_create_by({
-          message_id: in_reply_to_id,
-          email_conversation_id: conv.id,
-        })
       end
-      conv = in_reply_to_msg.email_conversation
-    else
-      conv = ::Office::EmailConversation.find_or_create_by({
-        subject: @message.subject,
+      @message.update_attributes({ email_conversation_id: conv.id })
+      conv.update_attributes({
+        state:       Conv::STATE_UNREAD,
+        latest_at:   the_mail.date || Time.now.to_datetime,
+        from_emails: ( conv.from_emails + the_mail.from ).uniq,
       })
-    end
-    @message.update_attributes({ email_conversation_id: conv.id })
-    conv.update_attributes({
-      state:       Conv::STATE_UNREAD,
-      latest_at:   the_mail.date || Time.now.to_datetime,
-      from_emails: ( conv.from_emails + the_mail.from ).uniq,
-    })
-    conv.add_tag( ::WpTag::INBOX )
-    conv_lead_tie = Office::EmailConversationLead.find_or_create_by({
-      lead_id: lead.id,
-      email_conversation_id: conv.id,
-    })
-
-
-    ## Actions & Filters
-    email_filters = Office::EmailFilter.active
-    email_filters.each do |filter|
-      if ( filter.from_regex.blank? ||     @message.from.match(                 filter.from_regex    ) ) &&
-         ( filter.from_exact.blank? ||     @message.from.downcase.include?(     filter.from_exact&.downcase ) ) &&
-         ( filter.body_exact.blank? ||     @message.part_html&.include?(         filter.body_exact    ) ) &&
-         ( filter.subject_regex.blank? ||  @message.subject.match(              filter.subject_regex ) ) &&
-         ( filter.subject_exact.blank? ||  @message.subject.downcase.include?(  filter.subject_exact&.downcase ) )
-
-        # || MiaTagger.analyze( @message.part_html, :is_spammy_recruite ).score > .5
-
-        puts! "applying filter #{filter} to conv #{conv}" if DEBUG
-
-        @message.apply_filter( filter )
+      conv.add_tag( ::WpTag::INBOX )
+      conv_lead_tie = Office::EmailConversationLead.find_or_create_by({
+        lead_id: lead.id,
+        email_conversation_id: conv.id,
+      })
+
+
+      ## Actions & Filters
+      email_filters = Office::EmailFilter.active
+      email_filters.each do |filter|
+        if ( filter.from_regex.blank? ||     @message.from.match(                 filter.from_regex    ) ) &&
+          ( filter.from_exact.blank? ||     @message.from.downcase.include?(     filter.from_exact&.downcase ) ) &&
+          ( filter.body_exact.blank? ||     @message.part_html&.include?(         filter.body_exact    ) ) &&
+          ( filter.subject_regex.blank? ||  @message.subject.match(              filter.subject_regex ) ) &&
+          ( filter.subject_exact.blank? ||  @message.subject.downcase.include?(  filter.subject_exact&.downcase ) )
+
+          # || MiaTagger.analyze( @message.part_html, :is_spammy_recruite ).score > .5
+
+          puts! "applying filter #{filter} to conv #{conv}" if DEBUG
+
+          @message.apply_filter( filter )
+        end
       end
-    end
 
-    stub.update_attributes({ state: ::Office::EmailMessageStub::STATE_PROCESSED })
+      stub.update_attributes({ state: ::Office::EmailMessageStub::STATE_PROCESSED })
 
-    ## Notification
-    conv = Conv.find( conv.id )
-    if conv.in_emailtag? WpTag::INBOX
-      out = ::Ishapi::ApplicationMailer.forwarder_notify( @message.id.to_s )
-      Rails.env.production? ? out.deliver_later : out.deliver_now
-    end
+      ## Notification
+      conv = Conv.find( conv.id )
+      if conv.in_emailtag? WpTag::INBOX
+        out = ::Ishapi::ApplicationMailer.forwarder_notify( @message.id.to_s )
+        Rails.env.production? ? out.deliver_later : out.deliver_now
+      end
 
   end
-
 end
 EIJ = Ishapi::EmailMessageIntakeJob

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ishapi
 version: !ruby/object:Gem::Version
-  version: 0.1.8.302
+  version: 0.1.8.303
 platform: ruby
 authors:
 - piousbox
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-08 00:00:00.000000000 Z
+date: 2023-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails