schleuder 3.5.3 → 3.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8aff8f8ecff4958e630158f6fc5811db96d26699ab0946d16b4fc430e2897387
-  data.tar.gz: daa4d6fbe4941a311dcfc54d5544b1771d809ca2f570f6060473b0f94f4bc577
+  metadata.gz: 648f81aa424a3214a9d03fb66e5b3e216c8990e092d1b5b18ab41a1f0a1e5fd3
+  data.tar.gz: 62a5a77189860b4228a2168690f3990d02e53ee4dfb6c458a0273b5407e8a2e7
 SHA512:
-  metadata.gz: b125a6734601100a1a768b9295848c4d3c1de0fb8336e7a6cf070972c78913f71c2ff52418a8a16cff705a546ca27de29155cae0c1f30736f4f8127bffe5b823
-  data.tar.gz: 55bd88009950e4dbc52a3f17f3566ee8f4bd7d43751237f19da514d00e8e3d2dc92817c15660b0d826bb8072cb537f3652cf5e93ebeca43fcc424faf33964978
+  metadata.gz: 97a79b18f6655bd6f9c56beded7527e061fd955433ac2e436a28176eef9d8903916b16a1b0ee22a51f600fbd56d4639b5fadd5a53f3df08d4ca156fc805c9cbb
+  data.tar.gz: 6e6b4ea2847974d64a3b38a0d99ec0bfa5bb824a36d566fa731889a711b88f127ad6314f9f44b8dd99937977dc0f9d9f2530419762fae527bb24e37b10517343

data/README.md

@@ -15,6 +15,7 @@ Requirements
 * gpgme
 * sqlite3
 * openssl
+* icu
 
 *If you use Debian buster or CentOS 7, please have a look at the [installation docs](https://schleuder.org/schleuder/docs/server-admins.html#installation). We do provide packages for those platforms, which simplify the installation a lot.*
 
@@ -22,7 +23,7 @@ Requirements
 
 On systems that base on Debian 10 ("buster"), install the dependencies via
 
-    apt-get install ruby-dev gnupg2 libgpgme-dev libsqlite3-dev libssl-dev build-essential
+    apt-get install ruby-dev gnupg2 libgpgme-dev libsqlite3-dev libssl-dev build-essential libicu-dev
 
 
 We **recommend** to also run a random number generator like [haveged](http://www.issihosts.com/haveged/). This ensures Schleuder won't be blocked by lacking entropy, which otherwise might happen especially during key generation.
@@ -47,15 +48,15 @@ Additionally these **rubygems** are required (will be installed automatically un
 Installing Schleuder
 ------------
 
-1. Download [the gem](https://schleuder.org/download/schleuder-3.5.3.gem) and [the OpenPGP-signature](https://schleuder.org/download/schleuder-3.5.3.gem.sig) and verify:
+1. Download [the gem](https://schleuder.org/download/schleuder-3.6.0.gem) and [the OpenPGP-signature](https://schleuder.org/download/schleuder-3.6.0.gem.sig) and verify:
    ```
    gpg --recv-key 0xB3D190D5235C74E1907EACFE898F2C91E2E6E1F3
-   gpg --verify schleuder-3.5.3.gem.sig
+   gpg --verify schleuder-3.6.0.gem.sig
    ```
 
 2. If all went well install the gem:
    ```
-   gem install schleuder-3.5.3.gem
+   gem install schleuder-3.6.0.gem
    ```
 
 3. Set up schleuder:
@@ -145,4 +146,4 @@ GNU GPL 3.0. Please see [LICENSE.txt](LICENSE.txt).
 Alternative Download
 --------------------
 
-Alternatively to the gem-files you can download the latest release as [a tarball](https://schleuder.org/download/schleuder-3.5.3.tar.gz) and [its OpenPGP-signature](https://schleuder.org/download/schleuder-3.5.3.tar.gz.sig).
+Alternatively to the gem-files you can download the latest release as [a tarball](https://schleuder.org/download/schleuder-3.6.0.tar.gz) and [its OpenPGP-signature](https://schleuder.org/download/schleuder-3.6.0.tar.gz.sig).

data/Rakefile

@@ -9,6 +9,8 @@ require_relative "lib/#{project}.rb"
 
 load "active_record/railties/databases.rake"
 
+puts @filename_gem
+
 # Configure ActiveRecord
 ActiveRecord::Tasks::DatabaseTasks.tap do |config|
   config.root = File.dirname(__FILE__)
@@ -89,7 +91,7 @@ end
 
 desc 'OpenPGP-sign gem and tarball'
 task :sign_tarball do
-  `gpg -u #{@gpguid} -b #{@filename_tarball}`
+  `gpg -v -u #{@gpguid} -b #{@filename_tarball}`
 end
 
 desc 'OpenPGP-sign gem'

data/db/migrate/20200118170110_add_set_reply_to_to_sender_and_munge_from.rb

@@ -0,0 +1,15 @@
+class AddSetReplyToToSenderAndMungeFrom < ActiveRecord::Migration
+  def up
+    if ! column_exists?(:lists, :set_reply_to_to_sender)
+      add_column :lists, :set_reply_to_to_sender, :boolean, default: false
+    end
+    if ! column_exists?(:lists, :munge_from)
+      add_column :lists, :munge_from, :boolean, default: false
+    end
+  end
+
+  def down
+    remove_column(:lists, :set_reply_to_to_sender)
+    remove_column(:lists, :munge_from)
+  end
+end

data/db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20190906194820) do
+ActiveRecord::Schema.define(version: 20200118170110) do
 
   create_table "lists", force: :cascade do |t|
     t.datetime "created_at"
@@ -46,6 +46,8 @@ ActiveRecord::Schema.define(version: 20190906194820) do
     t.integer  "logfiles_to_keep",                                        default: 2
     t.text     "internal_footer",                                         default: ""
     t.boolean  "include_autocrypt_header",                                default: true
+    t.boolean  "set_reply_to_to_sender",                                  default: false
+    t.boolean  "munge_from",                                              default: false
   end
 
   create_table "subscriptions", force: :cascade do |t|

data/etc/list-defaults.yml

@@ -131,3 +131,21 @@ forward_all_incoming_to_admins: false
 # Disabling this only works for signed e-mails; any e-mail that is unsigned
 # sent to the list is treated as coming from an unknown source
 deliver_selfsent: true
+
+# Set reply-to header to original sender's reply-to? 
+# Enabling this will set the reply-to-header of emails sent by schleuder
+# to the original sender's reply-to-header. If the original sender
+# did not supply a reply-to-header, the original from-header will be used.
+# This option can enabled for improved usability since this affect
+# mail client's reply-to button to reply to the original sender instead of
+# the whole list.
+set_reply_to_to_sender: false
+
+# Munge from-header?
+# Enabling this option will add the original sender to the from-header.
+# To avoid DMARC issues, we still use the list's address as from-address.
+# However the sender's address will be included as displayed name.
+# For example: "sender@sender.org via list@list.org" <list@list.org>
+# This option can enabled for improved usability since this affect
+# mail client's displayed name.
+munge_from: false

data/lib/schleuder/conf.rb

@@ -4,7 +4,13 @@ module Schleuder
   class Conf
     include Singleton
 
-    EMAIL_REGEXP = /\A.+@[[:alnum:]_.-]+\z/i
+    # since the regexp got only included into stdlib 2.2
+    # TODO: remove once 2.1 support dropped
+    if RUBY_VERSION < '2.2'
+      EMAIL_REGEXP = /\A[a-zA-Z0-9.!\#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*\z/
+    else
+      EMAIL_REGEXP = URI::MailTo::EMAIL_REGEXP
+    end
     # TODO: drop v3 keys and only accept length of 40
     FINGERPRINT_REGEXP = /\A(0x)?[a-f0-9]{32}([a-f0-9]{8})?\z/i
 

data/lib/schleuder/list.rb

@@ -67,6 +67,22 @@ module Schleuder
                 with: /\A[[:graph:]\s]*\z/i,
               }
 
+    # Some users find it quite confusing when they click "reply-to" and the mail client 
+    # doesn't reply to the sender of the mail but the whole mailing list. For those lists it can be
+    # considered to set this value to true. The recipients will then receive e-mails
+    # where the "reply-to" header will contain the reply-to address
+    # of the sender and thus reply to the sender when clicking "reply-to" in a client.
+    # If no "reply-to" is set, the "from"-header of the original sender will be used.
+    # The default is off.
+    validates :set_reply_to_to_sender, boolean: true
+
+    # Some users find it confusing when the "from" does not contain the original sender
+    # but the list address. For those lists it can be considered to set the munged header.
+    # This will result in a "from"-header like this: "originalsender@original.com via list@list.com"
+    # The default is off.
+    validates :munge_from, boolean: true
+
+
     default_scope { order(:email) }
 
     def self.configurable_attributes
@@ -367,7 +383,7 @@ module Schleuder
             next
           end
           
-          subscription.send_mail(mail)
+          subscription.send_mail(mail, incoming_mail)
           
         rescue => exc
           msg = I18n.t('errors.delivery_error',

data/lib/schleuder/mail/message.rb

@@ -205,17 +205,17 @@ module Mail
       @recipient.match(/-bounce@/).present? ||
           # Empty Return-Path
           self.return_path.to_s == '<>' ||
-          # Auto-Submitted exists and does not equal 'no' and:
-          #  - no cron header is present
-          #  - no Jenkins job notification header is present
-          # as these emails have the auto-submitted header.
-          ( self['Auto-Submitted'].present? && \
-            self['Auto-Submitted'].to_s.downcase != 'no' && \
-            !self['X-Cron-Env'].present? && \
-            !self['X-Jenkins-Job'].present? && \
-            self.subject.to_s !~ /\A\*\*\* SECURITY information.*\*\*\*\Z/)
+          bounced?
     end
 
+    def bounced?
+      @bounced ||= bounce_detected? || (error_status != "unknown")
+    end
+
+    def error_status
+      @error_status ||= detect_error_code
+    end
+ 
     def keywords
       return @keywords if @keywords
 
@@ -526,5 +526,79 @@ module Mail
         end
       end.join(' ')
     end
+
+    def detect_error_code
+      # Detects the error code of an email with different heuristics
+      # from: https://github.com/mailtop/bounce_email
+      
+      # Custom status codes
+      unicode_subject = self.subject.to_s
+      unicode_subject = unicode_subject.encode('utf-8') if unicode_subject.respond_to?(:encode)
+ 
+      return '97' if unicode_subject.match(/delayed/i)
+      return '98' if unicode_subject.match(/(unzulässiger|unerlaubter) anhang/i)
+      return '99' if unicode_subject.match(/auto.*reply|férias|ferias|Estarei ausente|estou ausente|vacation|vocation|(out|away).*office|on holiday|abwesenheits|autorespond|Automatische|eingangsbestätigung/i)
+
+      # Feedback-Type: abuse
+      return '96' if self.to_s.match(/Feedback-Type\: abuse/i)
+
+      if self.parts[1]
+        match_parts = self.parts[1].body.match(/(Status:.|550 |#)([245]\.[0-9]{1,3}\.[0-9]{1,3})/)
+        code = match_parts[2] if match_parts
+        return code if code
+      end
+
+      # Now try getting it from correct part of tmail
+      code = detect_bounce_status_code_from_text(self.body)
+      return code if code
+
+      # OK getting desperate so try getting code from entire email
+      code = detect_bounce_status_code_from_text(self.to_s)
+      code || 'unknown'
+    end
+
+    def bounce_detected?
+      # Detects bounces from different parts of the email without error status codes
+      # from: https://github.com/mailtop/bounce_email
+      return true if self.subject.to_s.match(/(returned|undelivered) mail|mail delivery( failed)?|(delivery )(status notification|failure)|failure notice|undeliver(able|ed)( mail)?|return(ing message|ed) to sender/i)
+      return true if self.subject.to_s.match(/auto.*reply|vacation|vocation|(out|away).*office|on holiday|abwesenheits|autorespond|Automatische|eingangsbestätigung/i)
+      return true if self['precedence'].to_s.match(/auto.*(reply|responder|antwort)/i)
+      return true if self.from.to_s.match(/^(MAILER-DAEMON|POSTMASTER)\@/i)
+      false
+    end
+
+    def detect_bounce_status_code_from_text(text)
+      # Parses a text and uses pattern matching to determines its error status (RFC 3463)
+      # from: https://github.com/mailtop/bounce_email
+      return "5.0.0" if text.match(/Status: 5\.0\.0/i)
+      return "5.1.1" if text.match(/no such (address|user)|Recipient address rejected|User unknown|does not like recipient|The recipient was unavailable to take delivery of the message|Sorry, no mailbox here by that name|invalid address|unknown user|unknown local part|user not found|invalid recipient|failed after I sent the message|did not reach the following recipient|nicht zugestellt werden|o pode ser entregue para um ou mais/i)
+      return "5.1.2" if text.match(/unrouteable mail domain|Esta casilla ha expirado por falta de uso|I couldn't find any host named/i)
+      if text.match(/mailbox is full|Mailbox quota (usage|disk) exceeded|quota exceeded|Over quota|User mailbox exceeds allowed size|Message rejected\. Not enough storage space|user has exhausted allowed storage space|too many messages on the server|mailbox is over quota|mailbox exceeds allowed size|excedeu a quota/i)
+        return "5.2.2" if text.match(/This is a permanent error||(Status: |)5\.2\.2/i)
+        return "4.2.2"
+      end
+      return "5.1.0" if text.match(/Address rejected/)
+      return "4.1.2" if text.match(/I couldn't find any host by that name/)
+      return "4.2.0" if text.match(/not yet been delivered/i)
+      return "5.1.1" if text.match(/mailbox unavailable|No such mailbox|RecipientNotFound|not found by SMTP address lookup|Status: 5\.1\.1/i)
+      return "5.2.3" if text.match(/Status: 5\.2\.3/i) # Too messages in folder
+      return "5.4.0" if text.match(/Status: 5\.4\.0/i) # too many hops
+      return "5.4.4" if text.match(/Unrouteable address/i)
+      return "4.4.7" if text.match(/retry timeout exceeded/i)
+      return "5.2.0" if text.match(/The account or domain may not exist, they may be blacklisted, or missing the proper dns entries./i)
+      return "5.5.4" if text.match(/554 TRANSACTION FAILED/i)
+      return "4.4.1" if text.match(/Status: 4.4.1|delivery temporarily suspended|wasn't able to establish an SMTP connection/i)
+      return "5.5.0" if text.match(/550 OU\-002|Mail rejected by Windows Live Hotmail for policy reasons/i)
+      return "5.1.2" if text.match(/PERM_FAILURE: DNS Error: Domain name not found/i)
+      return "4.2.0" if text.match(/Delivery attempts will continue to be made for/i)
+      return "5.5.4" if text.match(/554 delivery error:/i)
+      return "5.1.1" if text.match(/550-5.1.1|This Gmail user does not exist/i)
+      return "5.7.1" if text.match(/5.7.1 Your message.*?was blocked by ROTA DNSBL/i) # AA added
+      return "5.7.2" if text.match(/not have permission to post messages to the group/i)
+      return "5.3.2" if text.match(/Technical details of permanent failure|Too many bad recipients/i) && (text.match(/The recipient server did not accept our requests to connect/i) || text.match(/Connection was dropped by remote host/i) || text.match(/Could not initiate SMTP conversation/i)) # AA added
+      return "4.3.2" if text.match(/Technical details of temporary failure/i) && (text.match(/The recipient server did not accept our requests to connect/i) || text.match(/Connection was dropped by remote host/i) || text.match(/Could not initiate SMTP conversation/i)) # AA added
+      return "5.0.0" if text.match(/Delivery to the following recipient failed permanently/i) # AA added
+      return '5.2.3' if text.match(/account closed|account has been disabled or discontinued|mailbox not found|prohibited by administrator|access denied|account does not exist/i)
+    end
   end
 end

data/lib/schleuder/subscription.rb

@@ -10,6 +10,10 @@ module Schleuder
     validates :fingerprint, allow_blank: true, fingerprint: true
     validates :delivery_enabled, :admin, boolean: true
 
+    before_validation {
+      self.email = Mail::Address.new(self.email).address
+    }
+
     default_scope { order(:email) }
 
     scope :without_fingerprint, -> { where(fingerprint: [nil,'']) }
@@ -37,10 +41,10 @@ module Schleuder
       list.keys("0x#{self.fingerprint}").first
     end
 
-    def send_mail(mail)
+    def send_mail(mail, incoming_mail=nil)
       list.logger.debug "Preparing sending to #{self.inspect}"
 
-      mail = ensure_headers(mail)
+      mail = ensure_headers(mail, incoming_mail)
       gpg_opts = self.list.gpg_sign_options
 
       if self.key.blank?
@@ -66,9 +70,28 @@ module Schleuder
       mail.deliver
     end
 
-    def ensure_headers(mail)
+    def ensure_headers(mail, incoming_mail=nil)
       mail.to = self.email
-      mail.from = self.list.email
+      
+      if self.list.set_reply_to_to_sender? && ! incoming_mail.nil?
+        # If the option "set_reply_to_to_sender" is set to true, we will set the reply-to header 
+        # to the reply-to header given by the original email. If no reply-to header exists in the original email,
+        # the original senders email will be used as reply-to.
+        if ! incoming_mail.reply_to.nil?
+          mail.reply_to = incoming_mail.reply_to
+        else
+          mail.reply_to = incoming_mail.from
+        end
+      end
+
+      if self.list.munge_from? && ! incoming_mail.nil? 
+        # If the option "munge_from" is set to true, we will add the original senders' from-header to ours.
+        # We munge the from-header to avoid issues with DMARC.
+        mail.from = I18n.t("header_munging", from: incoming_mail.from.first, list: self.list.email, list_address: self.list.email)
+      else
+        mail.from = self.list.email
+      end
+
       mail.sender = self.list.bounce_address
       mail
     end

data/lib/schleuder/version.rb

@@ -1,3 +1,3 @@
 module Schleuder
-  VERSION = '3.5.3'
+  VERSION = '3.6.0'
 end

data/locales/de.yml

@@ -261,6 +261,7 @@ de:
   encryption_states:
     encrypted: "Verschlüsselt"
     unencrypted: "Unverschlüsselt"
+  header_munging: "%{from} über %{list} <%{list_address}>"
 
   activerecord:
     errors:

data/locales/en.yml

@@ -265,6 +265,7 @@ en:
   encryption_states:
     encrypted: "Encrypted"
     unencrypted: "Unencrypted"
+  header_munging: "%{from} via %{list} <%{list_address}>"
 
   activerecord:
     errors:

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: schleuder
 version: !ruby/object:Gem::Version
-  version: 3.5.3
+  version: 3.6.0
 platform: ruby
 authors:
 - schleuder dev team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-13 00:00:00.000000000 Z
+date: 2021-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gpgme
@@ -291,6 +291,7 @@ files:
 - db/migrate/20180110203100_add_sig_enc_to_headers_to_meta_defaults.rb
 - db/migrate/20180723173900_add_deliver_selfsent_to_list.rb
 - db/migrate/20190906194820_add_autocrypt_header_to_list.rb
+- db/migrate/20200118170110_add_set_reply_to_to_sender_and_munge_from.rb
 - db/schema.rb
 - etc/init.d/schleuder-api-daemon
 - etc/list-defaults.yml