RubyGems - ishapi - Versions diffs - 0.1.8.301 → 0.1.8.303 - Mend

ishapi 0.1.8.301 → 0.1.8.303

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (3) hide show

checksums.yaml +4 -4
data/app/jobs/ishapi/email_message_intake_job.rb +154 -214
metadata +2 -2

checksums.yaml CHANGED Viewed

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6229aa67b706172dea159f73ff6f5e45d9281c8cb64b9de70e384995f1a869c
-  data.tar.gz: af1216cc0d2d17bcb6a6e6b00e977e7b8ebca3768c6db71c7cd2568a3c21a4cf
+  metadata.gz: deb042effc76a3bac981e7d95c5ccbac9c1fe067b5afb6799f273a2fb061ba67
+  data.tar.gz: a497b8b725895391c965abe2680adb51d739958ca36172615263aa3fd6b83703
 SHA512:
-  metadata.gz: d16c460663a76a261be7a0ad5b29483cd71c800632bc990b5590206c8315aafea77abbb9b2ba65b8916ad3483e250cb019eeb6d3e22762a19389f1b4be775237
-  data.tar.gz: 9580c4aea9c855f492087492bbc266eb5db78344d6d966fdb929bd45157d5bbfe868d0f473594f579eed7cee33dd47eebd1c04d26a3c025ea75dc6c9ae7f56b2
+  metadata.gz: 206d685ea05a79c226a49d888e1ba2ced49e5ff32343999fe1a00e42c65eb8f139e17cc1dcb2bd6d0e7acbb83c257ef55205d75f318631c353a3aef22b6db143
+  data.tar.gz: ceebf0b9f54feb97eeac59cc655d65c12fc10957431d4821bf472e18822d97d80be31b0407f914a2a94e63119c62a5cf8a743e9f96556f088c481e90fbad3eff

data/app/jobs/ishapi/email_message_intake_job.rb CHANGED Viewed

@@ -9,243 +9,183 @@ class Ishapi::EmailMessageIntakeJob < Ishapi::ApplicationJob
   queue_as :default
-  ## For recursive parts of type `related`.
-  ## Content dispositions:
-  # "inline; creation-date=\"Tue, 11 Apr 2023 19:39:42 GMT\"; filename=image005.png; modification-date=\"Tue, 11 Apr 2023 19:47:53 GMT\"; size=14916",
-  #
-  ## Content Types:
-  # "application/pdf; name=\"Securities Forward Agreement -- HaulHub Inc -- Victor Pudeyev -- 2021-10-26.docx.pdf\""
-  # "image/jpeg; name=TX_DL_2.jpg"
-  # "image/png; name=image005.png"
-  # "multipart/alternative; boundary=_000_BL0PR10MB2913C560ADE059F0AB3A6D11829A9BL0PR10MB2913namp_",
-  # "text/html; charset=utf-8"
-  # "text/plain; charset=UTF-8"
-  # "text/calendar; charset=utf-8; method=REQUEST"
-  def churn_subpart message, part
-    if part.content_disposition&.include?('attachment')
-      ## @TODO: attachments !
-      ;
-    else
-      if part.content_type.include?("multipart/related") ||
-        part.content_type.include?("multipart/alternative")
-        part.parts.each do |subpart|
-          churn_subpart( message, subpart )
-        end
-      else
-        attachment = Office::EmailAttachment.new({
-          content:       part.decoded,
-          content_type:  part.content_type,
-          email_message: message,
-        })
-        attachment.save
+=begin
+  object_key = 'bg57j6u0j38b5ts86635fkqtjlucn5tvrm2hea81'
+  MsgStub.where({ object_key: object_key }).delete
-        if part.content_type.include?('text/html')
-          message.part_html = part.decoded
+  stub = MsgStub.create({ object_key: object_key })
+  id = stub.id
+=end
+  def perform id
+      stub = ::Office::EmailMessageStub.find id
+      if !Rails.env.test?
+        puts "Performing EmailMessageIntakeJob for object_key #{stub.object_key}"
+      end
+      if stub.state != ::Office::EmailMessageStub::STATE_PENDING
+        raise "This stub has already been processed: #{stub.id.to_s}."
+        return
+      end
+      client = Aws::S3::Client.new({
+        region:            ::S3_CREDENTIALS[:region_ses],
+        access_key_id:     ::S3_CREDENTIALS[:access_key_id_ses],
+        secret_access_key: ::S3_CREDENTIALS[:secret_access_key_ses],
+      })
-        elsif part.content_type.include?("text/plain")
-          message.part_txt = part.decoded
+      _mail              = client.get_object( bucket: ::S3_CREDENTIALS[:bucket_ses], key: stub.object_key ).body.read
+      the_mail           = Mail.new(_mail)
+      message_id         = the_mail.header['message-id'].decoded
+      in_reply_to_id     = the_mail.header['in-reply-to']&.to_s
+      email_inbox_tag_id = WpTag.emailtag(WpTag::INBOX).id
-        elsif part.content_type.include?("text/calendar")
-          ;
-        elsif part.content_type.include?("application/pdf")
-          ;
-        elsif part.content_type.include?("image/jpeg")
-          ;
-        elsif part.content_type.include?("image/png")
-          ;
+      if !the_mail.to
+        the_mail.to = [ 'NO-RECIPIENT' ]
+      end
-        else
-          puts! part.content_type, '444 No action for a part with this content_type'
-        end
-      end
-    end
-  end
+      subject   = ::Msg.strip_emoji the_mail.subject
+      subject ||= '(wco no subject)'
-  ## From: https://stackoverflow.com/questions/24672834/how-do-i-remove-emoji-from-string/24673322
-  def strip_emoji(text)
-    text = text.force_encoding('utf-8').encode
-    clean = ""
+      @message   = ::Office::EmailMessage.where( message_id: message_id ).first
+      @message ||= ::Office::EmailMessage.create({
+        raw: _mail,
-    # symbols & pics
-    regex = /[\u{1f300}-\u{1f5ff}]/
-    clean = text.gsub regex, ""
+        message_id:     message_id,
+        in_reply_to_id: in_reply_to_id,
-    # enclosed chars
-    regex = /[\u{2500}-\u{2BEF}]/ # I changed this to exclude chinese char
-    clean = clean.gsub regex, ""
+        object_key:  stub.object_key,
+        # object_path: stub.object_path,
-    # emoticons
-    regex = /[\u{1f600}-\u{1f64f}]/
-    clean = clean.gsub regex, ""
+        subject: subject,
+        date:    the_mail.date,
-    #dingbats
-    regex = /[\u{2702}-\u{27b0}]/
-    clean = clean.gsub regex, ""
-  end
+        from:  the_mail.from ? the_mail.from[0] : "nobody@unknown-doma.in",
+        froms: the_mail.from,
-  def perform id
-    stub = ::Office::EmailMessageStub.find id
-    if !Rails.env.test?
-      puts "Performing EmailMessageIntakeJob for object_key #{stub.object_key}"
-    end
-    if stub.state != ::Office::EmailMessageStub::STATE_PENDING
-      raise "This stub has already been processed: #{stub.id.to_s}."
-      return
-    end
-    client = Aws::S3::Client.new({
-      region:            ::S3_CREDENTIALS[:region_ses],
-      access_key_id:     ::S3_CREDENTIALS[:access_key_id_ses],
-      secret_access_key: ::S3_CREDENTIALS[:secret_access_key_ses],
-    })
-    _mail              = client.get_object( bucket: ::S3_CREDENTIALS[:bucket_ses], key: stub.object_key ).body.read
-    the_mail           = Mail.new(_mail)
-    message_id         = the_mail.header['message-id'].decoded
-    in_reply_to_id     = the_mail.header['in-reply-to']&.to_s
-    email_inbox_tag_id = WpTag.emailtag(WpTag::INBOX).id
-    if !the_mail.to
-      the_mail.to = [ 'NO-RECIPIENT' ]
-    end
-    subject   = strip_emoji the_mail.subject
-    subject ||= '(wco no subject)'
-    @message   = ::Office::EmailMessage.where( message_id: message_id ).first
-    @message ||= ::Office::EmailMessage.create({
-      raw: _mail,
-      message_id:     message_id,
-      in_reply_to_id: in_reply_to_id,
-      object_key:  stub.object_key,
-      # object_path: stub.object_path,
-      subject: subject,
-      date:    the_mail.date,
-      from:  the_mail.from ? the_mail.from[0] : "nobody@unknown-doma.in",
-      froms: the_mail.from,
-      to:  the_mail.to ? the_mail.to[0] : nil,
-      tos: the_mail.to,
-      cc:  the_mail.cc ? the_mail.cc[0] : nil,
-      ccs:  the_mail.cc,
-      # bccs: the_mail.bcc,
-    })
-    if !@message.persisted?
-      puts! @message.errors.full_messages, "Could not create email_message"
-    end
-    if the_mail.body.preamble.present?
-      @message.preamble = the_mail.body.preamble
-    end
-    if the_mail.body.epilogue.present?
-      @message.epilogue = the_mail.body.epilogue
-    end
-    ## Parts
-    the_mail.parts.each do |part|
-      churn_subpart( @message, part )
-    end
-    if the_mail.parts.length == 0
-      body = the_mail.body.decoded.encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
-      if the_mail.content_type&.include?('text/html')
-        @message.part_html = body
-      elsif the_mail.content_type&.include?('text/plain')
-        @message.part_txt = body
-      elsif the_mail.content_type.blank?
-        @message.part_txt = body
-      else
-        throw "mail body of unknown type: #{the_mail.content_type}"
-      end
-    end
-    ## Attachments
-    the_mail.attachments.each do |att|
-      photo = Photo.new({
-        content_type:      att.content_type.split(';')[0],
-        original_filename: att.content_type_parameters[:name],
-        image_data:        att.body.encoded,
-        email_message_id: @message.id,
+        to:  the_mail.to ? the_mail.to[0] : nil,
+        tos: the_mail.to,
+        cc:  the_mail.cc ? the_mail.cc[0] : nil,
+        ccs:  the_mail.cc,
+        # bccs: the_mail.bcc,
       })
-      photo.decode_base64_image
-      photo.save
-    end
-    ## Leadset, Lead
-    domain  = @message.from.split('@')[1] rescue 'unknown.domain'
-    leadset = Leadset.find_or_create_by( company_url: domain )
-    lead    = Lead.find_or_create_by( email: @message.from, m3_leadset_id: leadset.id )
-    the_mail.cc&.each do |cc|
-      domain  = cc.split('@')[1] rescue 'unknown.domain'
+      if !@message.persisted?
+        puts! @message.errors.full_messages, "Could not create email_message"
+      end
+      if the_mail.body.preamble.present?
+        @message.preamble = the_mail.body.preamble
+      end
+      if the_mail.body.epilogue.present?
+        @message.epilogue = the_mail.body.epilogue
+      end
+      ## Parts
+      the_mail.parts.each do |part|
+        @message.churn_subpart( part )
+      end
+      if the_mail.parts.length == 0
+        body = the_mail.body.decoded.encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
+        if the_mail.content_type&.include?('text/html')
+          @message.part_html = body
+        elsif the_mail.content_type&.include?('text/plain')
+          @message.part_txt = body
+        elsif the_mail.content_type.blank?
+          @message.part_txt = body
+        else
+          throw "mail body of unknown type: #{the_mail.content_type}"
+        end
+      end
+      ## Attachments
+      the_mail.attachments.each do |att|
+        photo = Photo.new({
+          content_type:      att.content_type.split(';')[0],
+          original_filename: att.content_type_parameters[:name],
+          image_data:        att.body.encoded,
+          email_message_id: @message.id,
+        })
+        photo.decode_base64_image
+        if photo.save
+          ;
+        else
+          attachment = Office::EmailAttachment.new({
+            content:       att.decoded,
+            content_type:  att.content_type,
+            email_message: @message,
+            filename:      att.filename,
+          })
+          attachment.save
+        end
+      end
+      ## Leadset, Lead
+      domain  = @message.from.split('@')[1] rescue 'unknown.domain'
       leadset = Leadset.find_or_create_by( company_url: domain )
-      Lead.find_or_create_by( email: cc, m3_leadset_id: leadset.id )
-    end
+      lead    = Lead.find_or_create_by( email: @message.from, m3_leadset_id: leadset.id )
+      the_mail.cc&.each do |cc|
+        domain  = cc.split('@')[1] rescue 'unknown.domain'
+        leadset = Leadset.find_or_create_by( company_url: domain )
+        Lead.find_or_create_by( email: cc, m3_leadset_id: leadset.id )
+      end
-    ## Conversation
-    if in_reply_to_id
-      in_reply_to_msg = ::Office::EmailMessage.where({ message_id: in_reply_to_id }).first
-      if !in_reply_to_msg
+      ## Conversation
+      if in_reply_to_id
+        in_reply_to_msg = ::Office::EmailMessage.where({ message_id: in_reply_to_id }).first
+        if !in_reply_to_msg
+          conv = ::Office::EmailConversation.find_or_create_by({
+            subject: @message.subject,
+          })
+          in_reply_to_msg = ::Office::EmailMessage.find_or_create_by({
+            message_id: in_reply_to_id,
+            email_conversation_id: conv.id,
+          })
+        end
+        conv = in_reply_to_msg.email_conversation
+      else
         conv = ::Office::EmailConversation.find_or_create_by({
           subject: @message.subject,
         })
-        in_reply_to_msg = ::Office::EmailMessage.find_or_create_by({
-          message_id: in_reply_to_id,
-          email_conversation_id: conv.id,
-        })
       end
-      conv = in_reply_to_msg.email_conversation
-    else
-      conv = ::Office::EmailConversation.find_or_create_by({
-        subject: @message.subject,
+      @message.update_attributes({ email_conversation_id: conv.id })
+      conv.update_attributes({
+        state:       Conv::STATE_UNREAD,
+        latest_at:   the_mail.date || Time.now.to_datetime,
+        from_emails: ( conv.from_emails + the_mail.from ).uniq,
       })
-    end
-    @message.update_attributes({ email_conversation_id: conv.id })
-    conv.update_attributes({
-      state:       Conv::STATE_UNREAD,
-      latest_at:   the_mail.date || Time.now.to_datetime,
-      from_emails: ( conv.from_emails + the_mail.from ).uniq,
-    })
-    conv.add_tag( ::WpTag::INBOX )
-    conv_lead_tie = Office::EmailConversationLead.find_or_create_by({
-      lead_id: lead.id,
-      email_conversation_id: conv.id,
-    })
-    ## Actions & Filters
-    email_filters = Office::EmailFilter.active
-    email_filters.each do |filter|
-      if ( filter.from_regex.blank? ||     @message.from.match(                 filter.from_regex    ) ) &&
-         ( filter.from_exact.blank? ||     @message.from.downcase.include?(     filter.from_exact&.downcase ) ) &&
-         ( filter.body_exact.blank? ||     @message.part_html&.include?(         filter.body_exact    ) ) &&
-         ( filter.subject_regex.blank? ||  @message.subject.match(              filter.subject_regex ) ) &&
-         ( filter.subject_exact.blank? ||  @message.subject.downcase.include?(  filter.subject_exact&.downcase ) )
-        # || MiaTagger.analyze( @message.part_html, :is_spammy_recruite ).score > .5
-        puts! "applying filter #{filter} to conv #{conv}" if DEBUG
-        @message.apply_filter( filter )
+      conv.add_tag( ::WpTag::INBOX )
+      conv_lead_tie = Office::EmailConversationLead.find_or_create_by({
+        lead_id: lead.id,
+        email_conversation_id: conv.id,
+      })
+      ## Actions & Filters
+      email_filters = Office::EmailFilter.active
+      email_filters.each do |filter|
+        if ( filter.from_regex.blank? ||     @message.from.match(                 filter.from_regex    ) ) &&
+          ( filter.from_exact.blank? ||     @message.from.downcase.include?(     filter.from_exact&.downcase ) ) &&
+          ( filter.body_exact.blank? ||     @message.part_html&.include?(         filter.body_exact    ) ) &&
+          ( filter.subject_regex.blank? ||  @message.subject.match(              filter.subject_regex ) ) &&
+          ( filter.subject_exact.blank? ||  @message.subject.downcase.include?(  filter.subject_exact&.downcase ) )
+          # || MiaTagger.analyze( @message.part_html, :is_spammy_recruite ).score > .5
+          puts! "applying filter #{filter} to conv #{conv}" if DEBUG
+          @message.apply_filter( filter )
+        end
       end
-    end
-    stub.update_attributes({ state: ::Office::EmailMessageStub::STATE_PROCESSED })
+      stub.update_attributes({ state: ::Office::EmailMessageStub::STATE_PROCESSED })
-    ## Notification
-    conv = Conv.find( conv.id )
-    if conv.in_emailtag? WpTag::INBOX
-      out = ::Ishapi::ApplicationMailer.forwarder_notify( @message.id.to_s )
-      Rails.env.production? ? out.deliver_later : out.deliver_now
-    end
+      ## Notification
+      conv = Conv.find( conv.id )
+      if conv.in_emailtag? WpTag::INBOX
+        out = ::Ishapi::ApplicationMailer.forwarder_notify( @message.id.to_s )
+        Rails.env.production? ? out.deliver_later : out.deliver_now
+      end
   end
 end
 EIJ = Ishapi::EmailMessageIntakeJob

metadata CHANGED Viewed

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ishapi
 version: !ruby/object:Gem::Version
-  version: 0.1.8.301
+  version: 0.1.8.303
 platform: ruby
 authors:
 - piousbox
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-08 00:00:00.000000000 Z
+date: 2023-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails