schleuder 3.5.3 → 4.0.2

data/lib/schleuder/list_builder.rb

@@ -20,7 +20,7 @@ module Schleuder
     end
 
     def run
-      Schleuder.logger.info "Building new list"
+      Schleuder.logger.info 'Building new list'
 
       if @listname.blank? || ! @listname.match(Conf::EMAIL_REGEXP)
         return [nil, {'email' => ["'#{@listname}' is not a valid email address"]}]
@@ -63,13 +63,13 @@ module Schleuder
 
     def gpg
       @gpg_ctx ||= begin
-        ENV["GNUPGHOME"] = @list_dir
+        ENV['GNUPGHOME'] = @list_dir
         GPGME::Ctx.new
       end
     end
 
     def create_key(list)
-      Schleuder.logger.info "Generating key-pair, this could take a while..."
+      Schleuder.logger.info 'Generating key-pair, this could take a while...'
       gpg.generate_key(key_params(list))
 
       # Get key without knowing the fingerprint yet.
@@ -88,14 +88,14 @@ module Schleuder
     def adduids(list, key)
       # Add UIDs for -owner and -request.
       [list.request_address, list.owner_address].each do |address|
-        err = key.adduid(list.email, address)
+        err = add_uid_to_key(list, address)
         if err.present?
           raise err
         end
       end
       # Go through list.key() to re-fetch the key from the keyring, otherwise
       # we don't see the new UIDs.
-      errors = list.key.set_primary_uid(list.email)
+      errors = set_primary_uid_of_key(list)
       if errors.present?
         raise errors
       end
@@ -135,5 +135,16 @@ module Schleuder
       end
     end
 
+    def set_primary_uid_of_key(list)
+      errors, _ = GPGME::Ctx.gpgcli("--quick-set-primary-uid #{list.email} '#{list.email} <#{list.email}>'")
+      errors.join
+    end
+
+    def add_uid_to_key(list, email)
+      # Specifying the key via fingerprint apparently doesn't work.
+      errors, _ = GPGME::Ctx.gpgcli("--quick-adduid #{list.email} '#{list.email} <#{email}>'")
+      errors.join
+    end
+
   end
 end

data/lib/schleuder/listlogger.rb

@@ -17,7 +17,7 @@ module Schleuder
       if logfiles_to_keep < 1
         logfiles_to_keep = list.class.column_defaults['logfiles_to_keep']
       end
-      suffix_now = Time.now.strftime("%Y%m%d").to_i
+      suffix_now = Time.now.strftime('%Y%m%d').to_i
       del_older_than = suffix_now - logfiles_to_keep
       Pathname.glob("#{list.logfile}.????????").each do |file|
         if file.basename.to_s.match(/\.([0-9]{8})$/)

data/lib/schleuder/logger.rb

@@ -7,13 +7,9 @@ module Schleuder
   class Logger < Syslog::Logger
     include LoggerNotifications
     def initialize
-      if RUBY_VERSION.to_f < 2.1
-        super('Schleuder')
-      else
-        super('Schleuder', Syslog::LOG_MAIL)
-      end
+      super('Schleuder', Syslog::LOG_MAIL)
       # We need some sender-address different from the superadmin-address.
-      @from = "#{`whoami`.chomp}@#{`hostname`.chomp}"
+      @from = "#{Etc.getlogin}@#{Socket.gethostname}"
       @adminaddresses = Conf.superadmin
       @level = ::Logger.const_get(Conf.log_level.upcase)
     end

data/lib/schleuder/mail/gpg/sign_part.rb

@@ -0,0 +1,33 @@
+module Mail
+  module Gpg
+    class SignPart < Mail::Part
+      # Copied verbatim from mail-gpg v.0.4.2. This code was changed in
+      # <https://github.com/jkraemer/mail-gpg/commit/5fded41ccee4a58f848a2f8e7bd53d11236f8984>,
+      # which breaks verifying some encapsulated (signed-then-encrypted)
+      # messages. See
+      # <https://github.com/jkraemer/mail-gpg/pull/40#issue-95776382> for
+      # details.
+      def self.verify_signature(plain_part, signature_part, options = {})
+        if !(signature_part.has_content_type? &&
+            ('application/pgp-signature' == signature_part.mime_type))
+          return false
+        end
+
+        # Work around the problem that plain_part.raw_source prefixes an
+        # erroneous CRLF, <https://github.com/mikel/mail/issues/702>.
+        if ! plain_part.raw_source.empty?
+          plaintext = [ plain_part.header.raw_source,
+                        "\r\n\r\n",
+                        plain_part.body.raw_source
+          ].join
+        else
+          plaintext = plain_part.encoded
+        end
+
+        signature = signature_part.body.encoded
+        GpgmeHelper.sign_verify(plaintext, signature, options)
+      end
+    end
+  end
+end
+

data/lib/schleuder/mail/message.rb

@@ -24,7 +24,9 @@ module Mail
     # Message#initialize.
     def setup
       if self.encrypted?
-        new = self.decrypt(verify: true)
+        # Specify 'loopback'-pinentry-mode to ensure that gnupg never-ever
+        # tries to interactively ask for a passphrase.
+        new = self.decrypt(verify: true, pinentry_mode: GPGME::PINENTRY_MODE_LOOPBACK)
         # Test if there's a signed multipart inside the ciphertext
         # ("encapsulated" format of pgp/mime).
         if encapsulated_signed?(new)
@@ -55,7 +57,7 @@ module Mail
       end
 
       # Delete the protected headers which might leak information.
-      if new.parts.first && new.parts.first.content_type == "text/rfc822-headers; protected-headers=v1"
+      if new.parts.first && new.parts.first.content_type == 'text/rfc822-headers; protected-headers=v1'
         new.parts.shift
       end
 
@@ -82,7 +84,7 @@ module Mail
 
       if self.protected_headers_subject.present?
         new_part = Mail::Part.new
-        new_part.content_type = "text/rfc822-headers; protected-headers=v1"
+        new_part.content_type = 'text/rfc822-headers; protected-headers=v1'
         new_part.body = "Subject: #{self.subject}\n"
         clean.add_part new_part
       end
@@ -141,7 +143,7 @@ module Mail
       when 1
         signatures.first
       else
-        raise "Multiple signatures found! Cannot handle!"
+        raise 'Multiple signatures found! Cannot handle!'
       end
     end
 
@@ -161,8 +163,10 @@ module Mail
     # subscription-assigned fingerprints are (should be) the ones of the
     # primary keys, so we need to look up the key.
     def signing_key
-      if signature.present?
-        @signing_key ||= list.keys(signature.fpr).first
+      @signing_key ||= begin
+        if signature.present?
+          list.keys(signature.fpr).first
+        end
       end
     end
 
@@ -205,17 +209,17 @@ module Mail
       @recipient.match(/-bounce@/).present? ||
           # Empty Return-Path
           self.return_path.to_s == '<>' ||
-          # Auto-Submitted exists and does not equal 'no' and:
-          #  - no cron header is present
-          #  - no Jenkins job notification header is present
-          # as these emails have the auto-submitted header.
-          ( self['Auto-Submitted'].present? && \
-            self['Auto-Submitted'].to_s.downcase != 'no' && \
-            !self['X-Cron-Env'].present? && \
-            !self['X-Jenkins-Job'].present? && \
-            self.subject.to_s !~ /\A\*\*\* SECURITY information.*\*\*\*\Z/)
+          bounced?
+    end
+
+    def bounced?
+      @bounced ||= bounce_detected? || (error_status != 'unknown')
     end
 
+    def error_status
+      @error_status ||= detect_error_code
+    end
+ 
     def keywords
       return @keywords if @keywords
 
@@ -224,23 +228,8 @@ module Mail
         return []
       end
 
-      @keywords = []
-      look_for_keywords = true
-      lines = part.decoded.lines.map do |line|
-        # TODO: Find multiline arguments (add-key). Currently add-key has to
-        # read the whole body and hope for the best.
-        if look_for_keywords && (m = line.match(/^x-([^:\s]*)[:\s]*(.*)/i))
-          command = m[1].strip.downcase
-          arguments = m[2].to_s.strip.downcase.split(/[,; ]{1,}/)
-          @keywords << [command, arguments]
-          nil
-        else
-          if look_for_keywords && line.match(/\S+/i)
-            look_for_keywords = false
-          end
-          line
-        end
-      end
+      @keywords, lines = extract_keywords(part.decoded.lines)
+      new_body = lines.join
 
       # Work around problems with re-encoding the body. If we delete the
       # content-transfer-encoding prior to re-assigning the body, and let Mail
@@ -249,7 +238,6 @@ module Mail
       part.content_transfer_encoding = nil
 
       # Set the right charset on the now parsed body
-      new_body = lines.compact.join
       part.charset = new_body.encoding.to_s
       part.body = new_body
 
@@ -292,19 +280,19 @@ module Mail
         if signing_key.present?
           signature_state = signature.to_s
         else
-          signature_state = I18n.t("signature_states.unknown", fingerprint: self.signature.fingerprint)
+          signature_state = I18n.t('signature_states.unknown', fingerprint: self.signature.fingerprint)
         end
       else
-        signature_state = I18n.t("signature_states.unsigned")
+        signature_state = I18n.t('signature_states.unsigned')
       end
       signature_state
     end
 
     def encryption_state
       if was_encrypted?
-        encryption_state = I18n.t("encryption_states.encrypted")
+        encryption_state = I18n.t('encryption_states.encrypted')
       else
-        encryption_state = I18n.t("encryption_states.unencrypted")
+        encryption_state = I18n.t('encryption_states.unencrypted')
       end
       encryption_state
     end
@@ -360,7 +348,7 @@ module Mail
         self['List-Help'] = '<https://schleuder.org/>'
 
         postmsg = if list.receive_admin_only
-                    "NO (Admins only)"
+                    'NO (Admins only)'
                   elsif list.receive_authenticated_only
                     "<mailto:#{list.email}> (Subscribers only)"
                   else
@@ -437,7 +425,6 @@ module Mail
       end
     end
 
-
     def attach_list_key!(list)
       filename = "#{list.email}.asc"
       self.add_file({
@@ -451,6 +438,40 @@ module Mail
 
     private
 
+
+    def extract_keywords(content_lines)
+      keywords = []
+      in_keyword_block = false
+      found_blank_line = false
+      content_lines.each_with_index do |line, i|
+        if match = line.match(/^x-([^:\s]*)[:\s]*(.*)/i)
+          keyword = match[1].strip.downcase
+          arguments = match[2].to_s.strip.downcase.split(/[,; ]{1,}/)
+          keywords << [keyword, arguments]
+          in_keyword_block = true
+
+          # Set this line to nil to have it stripped from the message.
+          content_lines[i] = nil
+        elsif line.blank? && keywords.any?
+          # Look for blank lines after the first keyword had been found.
+          # These might mark the end of the keywords-block — unless more keywords follow.
+          found_blank_line = true
+          # Swallow the line: before the actual content begins we want to drop blank lines.
+          content_lines[i] = nil
+          # Stop interpreting the following line as argument to the previous keyword.
+          in_keyword_block = false
+        elsif in_keyword_block == true
+          # Interpret line as arguments to the previous keyword.
+          keywords[-1][-1] += line.downcase.strip.split(/[,; ]{1,}/)
+          content_lines[i] = nil
+        elsif found_blank_line
+          # Any line that isn't blank and does not start with "x-" stops the keyword parsing.
+          break
+        end
+      end
+      [keywords, content_lines.compact]
+    end
+
     # mail.signed? throws an error if it finds
     # pgp boundaries, so we must use the Mail::Gpg
     # methods.
@@ -480,7 +501,7 @@ module Mail
     end
 
     def _add_subject_prefix(suffix)
-      attrib = "subject_prefix"
+      attrib = 'subject_prefix'
       if suffix
         attrib << "_#{suffix}"
       end
@@ -526,5 +547,79 @@ module Mail
         end
       end.join(' ')
     end
+
+    def detect_error_code
+      # Detects the error code of an email with different heuristics
+      # from: https://github.com/mailtop/bounce_email
+      
+      # Custom status codes
+      unicode_subject = self.subject.to_s
+      unicode_subject = unicode_subject.encode('utf-8') if unicode_subject.respond_to?(:encode)
+ 
+      return '97' if unicode_subject.match(/delayed/i)
+      return '98' if unicode_subject.match(/(unzulässiger|unerlaubter) anhang/i)
+      return '99' if unicode_subject.match(/auto.*reply|férias|ferias|Estarei ausente|estou ausente|vacation|vocation|(out|away).*office|on holiday|abwesenheits|autorespond|Automatische|eingangsbestätigung/i)
+
+      # Feedback-Type: abuse
+      return '96' if self.to_s.match(/Feedback-Type: abuse/i)
+
+      if self.parts[1]
+        match_parts = self.parts[1].body.match(/(Status:.|550 |#)([245]\.[0-9]{1,3}\.[0-9]{1,3})/)
+        code = match_parts[2] if match_parts
+        return code if code
+      end
+
+      # Now try getting it from correct part of tmail
+      code = detect_bounce_status_code_from_text(self.body)
+      return code if code
+
+      # OK getting desperate so try getting code from entire email
+      code = detect_bounce_status_code_from_text(self.to_s)
+      code || 'unknown'
+    end
+
+    def bounce_detected?
+      # Detects bounces from different parts of the email without error status codes
+      # from: https://github.com/mailtop/bounce_email
+      return true if self.subject.to_s.match(/(returned|undelivered) mail|mail delivery( failed)?|(delivery )(status notification|failure)|failure notice|undeliver(able|ed)( mail)?|return(ing message|ed) to sender/i)
+      return true if self.subject.to_s.match(/auto.*reply|vacation|vocation|(out|away).*office|on holiday|abwesenheits|autorespond|Automatische|eingangsbestätigung/i)
+      return true if self['precedence'].to_s.match(/auto.*(reply|responder|antwort)/i)
+      return true if self.from.to_s.match(/^(MAILER-DAEMON|POSTMASTER)@/i)
+      false
+    end
+
+    def detect_bounce_status_code_from_text(text)
+      # Parses a text and uses pattern matching to determines its error status (RFC 3463)
+      # from: https://github.com/mailtop/bounce_email
+      return '5.0.0' if text.match(/Status: 5\.0\.0/i)
+      return '5.1.1' if text.match(/no such (address|user)|Recipient address rejected|User unknown|does not like recipient|The recipient was unavailable to take delivery of the message|Sorry, no mailbox here by that name|invalid address|unknown user|unknown local part|user not found|invalid recipient|failed after I sent the message|did not reach the following recipient|nicht zugestellt werden|o pode ser entregue para um ou mais/i)
+      return '5.1.2' if text.match(/unrouteable mail domain|Esta casilla ha expirado por falta de uso|I couldn't find any host named/i)
+      if text.match(/mailbox is full|Mailbox quota (usage|disk) exceeded|quota exceeded|Over quota|User mailbox exceeds allowed size|Message rejected\. Not enough storage space|user has exhausted allowed storage space|too many messages on the server|mailbox is over quota|mailbox exceeds allowed size|excedeu a quota/i)
+        return '5.2.2' if text.match(/This is a permanent error||(Status: |)5\.2\.2/i)
+        return '4.2.2'
+      end
+      return '5.1.0' if text.match(/Address rejected/)
+      return '4.1.2' if text.match(/I couldn't find any host by that name/)
+      return '4.2.0' if text.match(/not yet been delivered/i)
+      return '5.1.1' if text.match(/mailbox unavailable|No such mailbox|RecipientNotFound|not found by SMTP address lookup|Status: 5\.1\.1/i)
+      return '5.2.3' if text.match(/Status: 5\.2\.3/i) # Too messages in folder
+      return '5.4.0' if text.match(/Status: 5\.4\.0/i) # too many hops
+      return '5.4.4' if text.match(/Unrouteable address/i)
+      return '4.4.7' if text.match(/retry timeout exceeded/i)
+      return '5.2.0' if text.match(/The account or domain may not exist, they may be blacklisted, or missing the proper dns entries./i)
+      return '5.5.4' if text.match(/554 TRANSACTION FAILED/i)
+      return '4.4.1' if text.match(/Status: 4.4.1|delivery temporarily suspended|wasn't able to establish an SMTP connection/i)
+      return '5.5.0' if text.match(/550 OU-002|Mail rejected by Windows Live Hotmail for policy reasons/i)
+      return '5.1.2' if text.match(/PERM_FAILURE: DNS Error: Domain name not found/i)
+      return '4.2.0' if text.match(/Delivery attempts will continue to be made for/i)
+      return '5.5.4' if text.match(/554 delivery error:/i)
+      return '5.1.1' if text.match(/550-5.1.1|This Gmail user does not exist/i)
+      return '5.7.1' if text.match(/5.7.1 Your message.*?was blocked by ROTA DNSBL/i) # AA added
+      return '5.7.2' if text.match(/not have permission to post messages to the group/i)
+      return '5.3.2' if text.match(/Technical details of permanent failure|Too many bad recipients/i) && (text.match(/The recipient server did not accept our requests to connect/i) || text.match(/Connection was dropped by remote host/i) || text.match(/Could not initiate SMTP conversation/i)) # AA added
+      return '4.3.2' if text.match(/Technical details of temporary failure/i) && (text.match(/The recipient server did not accept our requests to connect/i) || text.match(/Connection was dropped by remote host/i) || text.match(/Could not initiate SMTP conversation/i)) # AA added
+      return '5.0.0' if text.match(/Delivery to the following recipient failed permanently/i) # AA added
+      return '5.2.3' if text.match(/account closed|account has been disabled or discontinued|mailbox not found|prohibited by administrator|access denied|account does not exist/i)
+    end
   end
 end

data/lib/schleuder/runner.rb

@@ -4,12 +4,12 @@ module Schleuder
       error = setup_list(recipient)
       return error if error
 
-      logger.info "Parsing incoming email."
+      logger.info 'Parsing incoming email.'
 
       # is it valid utf-8?
       msg_scrubbed = false
       unless msg.valid_encoding?
-        logger.warn "Converting message due to invalid characters"
+        logger.warn 'Converting message due to invalid characters'
         detection = CharlockHolmes::EncodingDetector.detect(msg)
         begin
           msg = CharlockHolmes::Converter.convert(msg, detection[:encoding], 'UTF-8')
@@ -18,7 +18,7 @@ module Schleuder
           # so we scrub the invalid characters to be able to
           # at least parse the message somehow. Though this might
           # result in data loss.
-          logger.warn "Scrubbing message due to invalid characters"
+          logger.warn 'Scrubbing message due to invalid characters'
           msg = msg.scrub
           msg_scrubbed = true
         end
@@ -27,7 +27,7 @@ module Schleuder
       @mail = Mail.create_message_to_list(msg, recipient, list)
 
       if msg_scrubbed
-        @mail.add_pseudoheader(:note, I18n.t("pseudoheaders.scrubbed_message"))
+        @mail.add_pseudoheader(:note, I18n.t('pseudoheaders.scrubbed_message'))
       end
 
       error = run_filters('pre')
@@ -40,9 +40,10 @@ module Schleuder
       rescue GPGME::Error::BadPassphrase,
              GPGME::Error::DecryptFailed,
              GPGME::Error::NoData,
-             GPGME::Error::NoSecretKey
+             GPGME::Error::NoSecretKey,
+             GPGME::Error::Canceled
 
-        logger.warn "Decryption of incoming message failed."
+        logger.warn 'Decryption of incoming message failed.'
         return Errors::DecryptionFailed.new(list)
       end
 
@@ -50,29 +51,29 @@ module Schleuder
       return error if error
 
       if ! @mail.was_validly_signed?
-        logger.debug "Message was not validly signed, adding subject_prefix_in"
+        logger.debug 'Message was not validly signed, adding subject_prefix_in'
         @mail.add_subject_prefix_in!
       end
 
       if ! @mail.was_encrypted?
-        logger.debug "Message was not encrypted, skipping plugins"
+        logger.debug 'Message was not encrypted, skipping keyword-handlers'
       elsif @mail.was_validly_signed?
-        # Plugins
-        logger.debug "Message was encrypted and validly signed"
-        PluginRunners::ListPluginsRunner.run(list, @mail).compact
+        # run KeywordHandlers
+        logger.debug 'Message was encrypted and validly signed'
+        KeywordHandlersRunner.run(type: :list, list: list, mail: @mail).compact
       end
 
       # Don't send empty messages over the list.
       if @mail.empty?
-        logger.info "Message found empty, not sending it to list."
+        logger.info 'Message found empty, not sending it to list.'
         return Errors::MessageEmpty.new(@list)
       end
 
-      logger.debug "Adding subject_prefix"
+      logger.debug 'Adding subject_prefix'
       @mail.add_subject_prefix!
 
       # Subscriptions
-      logger.debug "Creating clean copy of message"
+      logger.debug 'Creating clean copy of message'
       copy = @mail.clean_copy(list.headers_to_meta.any?)
       list.send_to_subscriptions(copy, @mail)
       nil
@@ -105,10 +106,11 @@ module Schleuder
     end
 
     def filters_runner_pre_decryption
-      @filters_runner_pre_decryption ||= Filters::Runner.new(list,'pre')
+      @filters_runner_pre_decryption ||= Filters::Runner.new(list, 'pre')
     end
+
     def filters_runner_post_decryption
-      @filters_runner_post_decryption ||= Filters::Runner.new(list,'post')
+      @filters_runner_post_decryption ||= Filters::Runner.new(list, 'post')
     end
 
     def logger