schleuder 3.4.1 → 3.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3127723694c9320dbc25abc2bd7d5de2a0ca22d587c794bb36be5ae09fdf63f0
-  data.tar.gz: dfdcd502ca1498883cd931004034c7c4f8874271c06499bc39591cba407130a8
+  metadata.gz: 648f81aa424a3214a9d03fb66e5b3e216c8990e092d1b5b18ab41a1f0a1e5fd3
+  data.tar.gz: 62a5a77189860b4228a2168690f3990d02e53ee4dfb6c458a0273b5407e8a2e7
 SHA512:
-  metadata.gz: 31e3a5ed4c50043c7f0324f60595023cfc51766eca36aae32d326cffdcb81f09a3ad4798d4a96e43e99d67b768ab620dfe5b686ce95aeb5132f02e674cbc483a
-  data.tar.gz: bf93269f6737cfa57ce71474e15275e2379ea9c45cfc680698912d68674d0bd39a0d855ebbf09b59bdc00165948c81bc037b109012fd1b6b4a86f1664a9fcbb7
+  metadata.gz: 97a79b18f6655bd6f9c56beded7527e061fd955433ac2e436a28176eef9d8903916b16a1b0ee22a51f600fbd56d4639b5fadd5a53f3df08d4ca156fc805c9cbb
+  data.tar.gz: 6e6b4ea2847974d64a3b38a0d99ec0bfa5bb824a36d566fa731889a711b88f127ad6314f9f44b8dd99937977dc0f9d9f2530419762fae527bb24e37b10517343

data/README.md

@@ -15,14 +15,15 @@ Requirements
 * gpgme
 * sqlite3
 * openssl
+* icu
 
-*If you use Debian stretch or CentOS 7, please have a look at the [installation docs](https://schleuder.org/docs/#installation). We do provide packages for those platforms, which simplify the installation a lot.*
+*If you use Debian buster or CentOS 7, please have a look at the [installation docs](https://schleuder.org/schleuder/docs/server-admins.html#installation). We do provide packages for those platforms, which simplify the installation a lot.*
 
 *🛈 A note regarding Ubuntu: All Ubuntu versions up to and including 17.10 don't meet the requirements with their packaged versions of gnupg! To run Schleuder on Ubuntu you currently have to install a more recent version of gnupg manually. Only Ubuntu 18.04 ("bionic") provides modern enough versions of Schleuder's requirements.*
 
-On systems that base on Debian 9 ("stretch"), install the dependencies via
+On systems that base on Debian 10 ("buster"), install the dependencies via
 
-    apt-get install ruby-dev gnupg2 libgpgme-dev libsqlite3-dev libssl-dev build-essential
+    apt-get install ruby-dev gnupg2 libgpgme-dev libsqlite3-dev libssl-dev build-essential libicu-dev
 
 
 We **recommend** to also run a random number generator like [haveged](http://www.issihosts.com/haveged/). This ensures Schleuder won't be blocked by lacking entropy, which otherwise might happen especially during key generation.
@@ -47,15 +48,15 @@ Additionally these **rubygems** are required (will be installed automatically un
 Installing Schleuder
 ------------
 
-1. Download [the gem](https://schleuder.org/download/schleuder-3.4.1.gem) and [the OpenPGP-signature](https://schleuder.org/download/schleuder-3.4.1.gem.sig) and verify:
+1. Download [the gem](https://schleuder.org/download/schleuder-3.6.0.gem) and [the OpenPGP-signature](https://schleuder.org/download/schleuder-3.6.0.gem.sig) and verify:
    ```
    gpg --recv-key 0xB3D190D5235C74E1907EACFE898F2C91E2E6E1F3
-   gpg --verify schleuder-3.4.1.gem.sig
+   gpg --verify schleuder-3.6.0.gem.sig
    ```
 
 2. If all went well install the gem:
    ```
-   gem install schleuder-3.4.1.gem
+   gem install schleuder-3.6.0.gem
    ```
 
 3. Set up schleuder:
@@ -65,7 +66,7 @@ Installing Schleuder
   This creates necessary directories, copies example configs, etc. If you see errors about missing write permissions please follow the advice given.
 
 
-For further information on setup and configuration please read <https://schleuder.org/docs/#setup>.
+For further information on setup and configuration please read <https://schleuder.org/schleuder/docs/server-admins.html>.
 
 
 Command line usage
@@ -112,7 +113,7 @@ To execute the test suite run:
 
     bundle exec rspec
 
-Please note: Some of the specs use 'pgrep'. On systems that base on Debian 9 ("stretch") install it via 
+Please note: Some of the specs use 'pgrep'. On systems that base on Debian 10 ("buster") install it via 
 
     apt-get install procps
 
@@ -145,4 +146,4 @@ GNU GPL 3.0. Please see [LICENSE.txt](LICENSE.txt).
 Alternative Download
 --------------------
 
-Alternatively to the gem-files you can download the latest release as [a tarball](https://schleuder.org/download/schleuder-3.4.1.tar.gz) and [its OpenPGP-signature](https://schleuder.org/download/schleuder-3.4.1.tar.gz.sig).
+Alternatively to the gem-files you can download the latest release as [a tarball](https://schleuder.org/download/schleuder-3.6.0.tar.gz) and [its OpenPGP-signature](https://schleuder.org/download/schleuder-3.6.0.tar.gz.sig).

data/Rakefile

@@ -3,12 +3,14 @@ require_relative "lib/#{project}.rb"
 
 @version = Schleuder::VERSION
 @tagname = "#{project}-#{@version}"
-@gpguid = 'team@schleuder.org'
+@gpguid = 'B3D190D5235C74E1907EACFE898F2C91E2E6E1F3'
 @filename_gem = "#{@tagname}.gem"
 @filename_tarball = "#{@tagname}.tar.gz"
 
 load "active_record/railties/databases.rake"
 
+puts @filename_gem
+
 # Configure ActiveRecord
 ActiveRecord::Tasks::DatabaseTasks.tap do |config|
   config.root = File.dirname(__FILE__)
@@ -89,7 +91,7 @@ end
 
 desc 'OpenPGP-sign gem and tarball'
 task :sign_tarball do
-  `gpg -u #{@gpguid} -b #{@filename_tarball}`
+  `gpg -v -u #{@gpguid} -b #{@filename_tarball}`
 end
 
 desc 'OpenPGP-sign gem'
@@ -110,7 +112,7 @@ end
 desc 'Publish gem-file to rubygems.org'
 task :publish_gem do
   puts "Really push #{@filename_gem} to rubygems.org? [yN]"
-  if gets.match(/^y/i)
+  if $stdin.gets.match(/^y/i)
     puts "Pushing..."
     `gem push #{@filename_gem}`
   else
@@ -132,7 +134,7 @@ desc 'Check if version-tag already exists'
 task :check_version do
   # Check if Schleuder::VERSION has been updated since last release
   if `git tag`.match?(/^#{@tagname}$/)
-    $stderr.puts "Warning: Tag '#{@tagname}' already exists. Did you forget to update #{project}/version.rb?"
+    $stderr.puts "Warning: Tag '#{@tagname}' already exists. Did you forget to update lib/#{project}/version.rb?"
     $stderr.print "Delete tag to continue? [yN] "
     if $stdin.gets.match(/^y/i)
       `git tag -d #{@tagname}`

data/db/migrate/20180723173900_add_deliver_selfsent_to_list.rb

@@ -0,0 +1,11 @@
+class AddDeliverSelfsentToList < ActiveRecord::Migration
+  def up
+    if ! column_exists?(:lists, :deliver_selfsent)
+      add_column :lists, :deliver_selfsent, :boolean, default: true
+    end
+  end
+
+  def down
+    remove_column(:lists, :deliver_selfsent)
+  end
+end

data/db/migrate/20190906194820_add_autocrypt_header_to_list.rb

@@ -0,0 +1,11 @@
+class AddAutocryptHeaderToList < ActiveRecord::Migration
+  def up
+    if ! column_exists?(:lists, :include_autocrypt_header)
+      add_column :lists, :include_autocrypt_header, :boolean, default: true
+    end
+  end
+
+  def down
+    remove_column(:lists, :include_autocrypt_header)
+  end
+end

data/db/migrate/20200118170110_add_set_reply_to_to_sender_and_munge_from.rb

@@ -0,0 +1,15 @@
+class AddSetReplyToToSenderAndMungeFrom < ActiveRecord::Migration
+  def up
+    if ! column_exists?(:lists, :set_reply_to_to_sender)
+      add_column :lists, :set_reply_to_to_sender, :boolean, default: false
+    end
+    if ! column_exists?(:lists, :munge_from)
+      add_column :lists, :munge_from, :boolean, default: false
+    end
+  end
+
+  def down
+    remove_column(:lists, :set_reply_to_to_sender)
+    remove_column(:lists, :munge_from)
+  end
+end

data/db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180110203100) do
+ActiveRecord::Schema.define(version: 20200118170110) do
 
   create_table "lists", force: :cascade do |t|
     t.datetime "created_at"
@@ -37,6 +37,7 @@ ActiveRecord::Schema.define(version: 20180110203100) do
     t.boolean  "keep_msgid",                                              default: true
     t.boolean  "bounces_drop_all",                                        default: false
     t.boolean  "bounces_notify_admins",                                   default: true
+    t.boolean  "deliver_selfsent",                                        default: true
     t.boolean  "include_list_headers",                                    default: true
     t.boolean  "include_openpgp_header",                                  default: true
     t.integer  "max_message_size_kb",                                     default: 10240
@@ -44,6 +45,9 @@ ActiveRecord::Schema.define(version: 20180110203100) do
     t.boolean  "forward_all_incoming_to_admins",                          default: false
     t.integer  "logfiles_to_keep",                                        default: 2
     t.text     "internal_footer",                                         default: ""
+    t.boolean  "include_autocrypt_header",                                default: true
+    t.boolean  "set_reply_to_to_sender",                                  default: false
+    t.boolean  "munge_from",                                              default: false
   end
 
   create_table "subscriptions", force: :cascade do |t|

data/etc/list-defaults.yml

@@ -93,6 +93,9 @@ bounces_drop_on_headers:
 # Send a notice to the list-admins whenever an email is bounced or dropped?
 bounces_notify_admins: true
 
+# Include Autocrypt header into emails?
+include_autocrypt_header: true
+
 # Include RFC-compliant List-* Headers into emails?
 include_list_headers: true
 
@@ -123,3 +126,26 @@ language: en
 # Forward a raw copy of all incoming emails to the list-admins?
 # Mainly useful for debugging.
 forward_all_incoming_to_admins: false
+
+# Should e-mails be delivered to the original subscribed sender?
+# Disabling this only works for signed e-mails; any e-mail that is unsigned
+# sent to the list is treated as coming from an unknown source
+deliver_selfsent: true
+
+# Set reply-to header to original sender's reply-to? 
+# Enabling this will set the reply-to-header of emails sent by schleuder
+# to the original sender's reply-to-header. If the original sender
+# did not supply a reply-to-header, the original from-header will be used.
+# This option can enabled for improved usability since this affect
+# mail client's reply-to button to reply to the original sender instead of
+# the whole list.
+set_reply_to_to_sender: false
+
+# Munge from-header?
+# Enabling this option will add the original sender to the from-header.
+# To avoid DMARC issues, we still use the list's address as from-address.
+# However the sender's address will be included as displayed name.
+# For example: "sender@sender.org via list@list.org" <list@list.org>
+# This option can enabled for improved usability since this affect
+# mail client's displayed name.
+munge_from: false

data/lib/schleuder.rb

@@ -1,3 +1,10 @@
+# default to UTF-8 encoding as early as possible for external
+# data.
+#
+# this should ensure we are able to parse most incoming
+# plain text mails in different charsets.
+Encoding.default_external = Encoding::UTF_8
+
 # Stdlib
 require 'fileutils'
 require 'singleton'
@@ -61,6 +68,9 @@ ENV["SCHLEUDER_CONFIG"] ||= '/etc/schleuder/schleuder.yml'
 ENV["SCHLEUDER_LIST_DEFAULTS"] ||= '/etc/schleuder/list-defaults.yml'
 ENV["SCHLEUDER_ENV"] ||= 'production'
 ENV["SCHLEUDER_ROOT"] = rootdir.to_s
+# Ensure that gnupg never-ever tries to ask for a passphrase.
+ENV["GPG_TTY"] = "/nonexistant-#{rand}"
+ENV["DISPLAY"] = nil
 
 GPGME::Ctx.set_gpg_path_from_env
 GPGME::Ctx.check_gpg_version

data/lib/schleuder/cli.rb

@@ -1,6 +1,7 @@
 require 'thor'
 require 'yaml'
 require 'gpgme'
+require 'charlock_holmes'
 
 require_relative '../schleuder'
 require 'schleuder/cli/subcommand_fix'

data/lib/schleuder/conf.rb

@@ -4,7 +4,13 @@ module Schleuder
   class Conf
     include Singleton
 
-    EMAIL_REGEXP = /\A.+@[[:alnum:]_.-]+\z/i
+    # since the regexp got only included into stdlib 2.2
+    # TODO: remove once 2.1 support dropped
+    if RUBY_VERSION < '2.2'
+      EMAIL_REGEXP = /\A[a-zA-Z0-9.!\#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*\z/
+    else
+      EMAIL_REGEXP = URI::MailTo::EMAIL_REGEXP
+    end
     # TODO: drop v3 keys and only accept length of 40
     FINGERPRINT_REGEXP = /\A(0x)?[a-f0-9]{32}([a-f0-9]{8})?\z/i
 

data/lib/schleuder/gpgme/key.rb

@@ -57,6 +57,10 @@ module GPGME
       "#{self.to_s}\n\n#{export(armor: true).read}"
     end
 
+    def minimal
+      export(minimal: true).to_s
+    end
+
     # Force encoding, some databases save "ASCII-8BIT" as binary data.
     alias_method :orig_fingerprint, :fingerprint
     def fingerprint

data/lib/schleuder/list.rb

@@ -19,6 +19,7 @@ module Schleuder
         :receive_admin_only,
         :keep_msgid,
         :bounces_drop_all,
+        :deliver_selfsent,
         :bounces_notify_admins,
         :include_list_headers,
         :include_openpgp_header,
@@ -66,6 +67,22 @@ module Schleuder
                 with: /\A[[:graph:]\s]*\z/i,
               }
 
+    # Some users find it quite confusing when they click "reply-to" and the mail client 
+    # doesn't reply to the sender of the mail but the whole mailing list. For those lists it can be
+    # considered to set this value to true. The recipients will then receive e-mails
+    # where the "reply-to" header will contain the reply-to address
+    # of the sender and thus reply to the sender when clicking "reply-to" in a client.
+    # If no "reply-to" is set, the "from"-header of the original sender will be used.
+    # The default is off.
+    validates :set_reply_to_to_sender, boolean: true
+
+    # Some users find it confusing when the "from" does not contain the original sender
+    # but the list address. For those lists it can be considered to set the munged header.
+    # This will result in a "from"-header like this: "originalsender@original.com via list@list.com"
+    # The default is off.
+    validates :munge_from, boolean: true
+
+
     default_scope { order(:email) }
 
     def self.configurable_attributes
@@ -146,6 +163,16 @@ module Schleuder
       key.armored
     end
 
+    def key_minimal_base64_encoded(fingerprint=self.fingerprint)
+      key = keys(fingerprint).first
+      
+      if key.blank?
+        return false
+      end
+      
+      Base64.strict_encode64(key.minimal)
+    end
+
     def check_keys
       now = Time.now
       checkdate = now + (60 * 60 * 24 * 14) # two weeks
@@ -340,16 +367,24 @@ module Schleuder
       true
     end
 
-    def send_to_subscriptions(mail)
+    def send_to_subscriptions(mail, incoming_mail=nil)
       logger.debug "Sending to subscriptions."
       mail.add_internal_footer!
       self.subscriptions.each do |subscription|
         begin
-          if subscription.delivery_enabled
-            subscription.send_mail(mail)
-          else
+          
+          if ! subscription.delivery_enabled
             logger.info "Not sending to #{subscription.email}: delivery is disabled."
+            next
+          end
+          
+          if ! self.deliver_selfsent && incoming_mail.was_validly_signed? && ( subscription == incoming_mail.signer )
+            logger.info "Not sending to #{subscription.email}: delivery of self sent is disabled."
+            next
           end
+          
+          subscription.send_mail(mail, incoming_mail)
+          
         rescue => exc
           msg = I18n.t('errors.delivery_error',
                        { email: subscription.email, error: exc.to_s })

data/lib/schleuder/logger_notifications.rb

@@ -42,6 +42,8 @@ module Schleuder
             gpg_opts.merge!(encrypt: true, keys: { address => key.fingerprint })
           end
           mail.gpg gpg_opts
+
+          mail.header['List-Id'] = "<#{@list.email.gsub('@', '.')}>"
         end
         mail.deliver
       end

data/lib/schleuder/mail/message.rb

@@ -17,6 +17,7 @@ module Mail
     attr_accessor :original_message
     attr_accessor :list
     attr_accessor :protected_headers_subject
+    attr_writer :dynamic_pseudoheaders
 
     # TODO: This should be in initialize(), but I couldn't understand the
     # strange errors about wrong number of arguments when overriding
@@ -44,9 +45,7 @@ module Mail
       # might be gone (e.g. request-keywords that delete subscriptions or
       # keys).
       new.signer
-      self.dynamic_pseudoheaders.each do |str|
-        new.add_pseudoheader(str)
-      end
+      new.dynamic_pseudoheaders = self.dynamic_pseudoheaders.dup
 
       # Store previously protected subject for later access.
       # mail-gpg pulls headers from the decrypted mime parts "up" into the main
@@ -206,13 +205,17 @@ module Mail
       @recipient.match(/-bounce@/).present? ||
           # Empty Return-Path
           self.return_path.to_s == '<>' ||
-          # Auto-Submitted exists and does not equal 'no' and no cron header
-          # present, as cron emails have the auto-submitted header.
-          ( self['Auto-Submitted'].present? && \
-            self['Auto-Submitted'].to_s.downcase != 'no' && \
-            !self['X-Cron-Env'].present?)
+          bounced?
+    end
+
+    def bounced?
+      @bounced ||= bounce_detected? || (error_status != "unknown")
     end
 
+    def error_status
+      @error_status ||= detect_error_code
+    end
+ 
     def keywords
       return @keywords if @keywords
 
@@ -244,11 +247,11 @@ module Mail
       # decide itself how to encode, it works. If we don't, some
       # character-sequences are not properly re-encoded.
       part.content_transfer_encoding = nil
-      # Make the converted strings (now UTF-8) match what mime-part's headers say,
-      # fall back to US-ASCII if none is set.
-      # https://tools.ietf.org/html/rfc2046#section-4.1.2
-      # -> Default charset is US-ASCII
-      part.body = lines.compact.join.encode(part.charset||'US-ASCII')
+
+      # Set the right charset on the now parsed body
+      new_body = lines.compact.join
+      part.charset = new_body.encoding.to_s
+      part.body = new_body
 
       @keywords
     end
@@ -266,20 +269,17 @@ module Mail
     end
 
     def add_pseudoheader(string_or_key, value=nil)
-      @dynamic_pseudoheaders ||= []
-      if value.present?
-        @dynamic_pseudoheaders << make_pseudoheader(string_or_key, value)
-      else
-        @dynamic_pseudoheaders << string_or_key.to_s
-      end
+      dynamic_pseudoheaders << make_pseudoheader(string_or_key, value)
     end
 
     def make_pseudoheader(key, value)
-      "#{key.to_s.camelize}: #{value.to_s}"
+      output = "#{key.to_s.camelize}: #{value.to_s}"
+      # wrap lines after 76 with 2 indents
+      output.gsub(/(.{1,76})( +|$)\n?/, "  \\1\n").chomp.lstrip
     end
 
     def dynamic_pseudoheaders
-      @dynamic_pseudoheaders || []
+      @dynamic_pseudoheaders ||= []
     end
 
     def signature_state
@@ -330,7 +330,8 @@ module Mail
     end
 
     def pseudoheaders(list)
-      (standard_pseudoheaders(list) + dynamic_pseudoheaders).flatten.join("\n") + "\n"
+      separator = '------------------------------------------------------------------------------'
+      (standard_pseudoheaders(list) + dynamic_pseudoheaders).flatten.join("\n") + "\n" + separator + "\n"
     end
 
     def add_msgids(list, orig)
@@ -345,6 +346,14 @@ module Mail
     end
 
     def add_list_headers(list)
+      if list.include_autocrypt_header
+        # Inject whitespaces, to let Mail break the string at these points
+        # leading to correct wrapping.
+        keydata = list.key_minimal_base64_encoded.gsub(/(.{78})/, '\1 ')
+        
+        self['Autocrypt'] = "addr=#{list.email}; prefer-encrypt=mutual; keydata=#{keydata}"
+      end
+
       if list.include_list_headers
         self['List-Id'] = "<#{list.email.gsub('@', '.')}>"
         self['List-Owner'] = "<mailto:#{list.owner_address}> (Use list's public key)"
@@ -517,5 +526,79 @@ module Mail
         end
       end.join(' ')
     end
+
+    def detect_error_code
+      # Detects the error code of an email with different heuristics
+      # from: https://github.com/mailtop/bounce_email
+      
+      # Custom status codes
+      unicode_subject = self.subject.to_s
+      unicode_subject = unicode_subject.encode('utf-8') if unicode_subject.respond_to?(:encode)
+ 
+      return '97' if unicode_subject.match(/delayed/i)
+      return '98' if unicode_subject.match(/(unzulässiger|unerlaubter) anhang/i)
+      return '99' if unicode_subject.match(/auto.*reply|férias|ferias|Estarei ausente|estou ausente|vacation|vocation|(out|away).*office|on holiday|abwesenheits|autorespond|Automatische|eingangsbestätigung/i)
+
+      # Feedback-Type: abuse
+      return '96' if self.to_s.match(/Feedback-Type\: abuse/i)
+
+      if self.parts[1]
+        match_parts = self.parts[1].body.match(/(Status:.|550 |#)([245]\.[0-9]{1,3}\.[0-9]{1,3})/)
+        code = match_parts[2] if match_parts
+        return code if code
+      end
+
+      # Now try getting it from correct part of tmail
+      code = detect_bounce_status_code_from_text(self.body)
+      return code if code
+
+      # OK getting desperate so try getting code from entire email
+      code = detect_bounce_status_code_from_text(self.to_s)
+      code || 'unknown'
+    end
+
+    def bounce_detected?
+      # Detects bounces from different parts of the email without error status codes
+      # from: https://github.com/mailtop/bounce_email
+      return true if self.subject.to_s.match(/(returned|undelivered) mail|mail delivery( failed)?|(delivery )(status notification|failure)|failure notice|undeliver(able|ed)( mail)?|return(ing message|ed) to sender/i)
+      return true if self.subject.to_s.match(/auto.*reply|vacation|vocation|(out|away).*office|on holiday|abwesenheits|autorespond|Automatische|eingangsbestätigung/i)
+      return true if self['precedence'].to_s.match(/auto.*(reply|responder|antwort)/i)
+      return true if self.from.to_s.match(/^(MAILER-DAEMON|POSTMASTER)\@/i)
+      false
+    end
+
+    def detect_bounce_status_code_from_text(text)
+      # Parses a text and uses pattern matching to determines its error status (RFC 3463)
+      # from: https://github.com/mailtop/bounce_email
+      return "5.0.0" if text.match(/Status: 5\.0\.0/i)
+      return "5.1.1" if text.match(/no such (address|user)|Recipient address rejected|User unknown|does not like recipient|The recipient was unavailable to take delivery of the message|Sorry, no mailbox here by that name|invalid address|unknown user|unknown local part|user not found|invalid recipient|failed after I sent the message|did not reach the following recipient|nicht zugestellt werden|o pode ser entregue para um ou mais/i)
+      return "5.1.2" if text.match(/unrouteable mail domain|Esta casilla ha expirado por falta de uso|I couldn't find any host named/i)
+      if text.match(/mailbox is full|Mailbox quota (usage|disk) exceeded|quota exceeded|Over quota|User mailbox exceeds allowed size|Message rejected\. Not enough storage space|user has exhausted allowed storage space|too many messages on the server|mailbox is over quota|mailbox exceeds allowed size|excedeu a quota/i)
+        return "5.2.2" if text.match(/This is a permanent error||(Status: |)5\.2\.2/i)
+        return "4.2.2"
+      end
+      return "5.1.0" if text.match(/Address rejected/)
+      return "4.1.2" if text.match(/I couldn't find any host by that name/)
+      return "4.2.0" if text.match(/not yet been delivered/i)
+      return "5.1.1" if text.match(/mailbox unavailable|No such mailbox|RecipientNotFound|not found by SMTP address lookup|Status: 5\.1\.1/i)
+      return "5.2.3" if text.match(/Status: 5\.2\.3/i) # Too messages in folder
+      return "5.4.0" if text.match(/Status: 5\.4\.0/i) # too many hops
+      return "5.4.4" if text.match(/Unrouteable address/i)
+      return "4.4.7" if text.match(/retry timeout exceeded/i)
+      return "5.2.0" if text.match(/The account or domain may not exist, they may be blacklisted, or missing the proper dns entries./i)
+      return "5.5.4" if text.match(/554 TRANSACTION FAILED/i)
+      return "4.4.1" if text.match(/Status: 4.4.1|delivery temporarily suspended|wasn't able to establish an SMTP connection/i)
+      return "5.5.0" if text.match(/550 OU\-002|Mail rejected by Windows Live Hotmail for policy reasons/i)
+      return "5.1.2" if text.match(/PERM_FAILURE: DNS Error: Domain name not found/i)
+      return "4.2.0" if text.match(/Delivery attempts will continue to be made for/i)
+      return "5.5.4" if text.match(/554 delivery error:/i)
+      return "5.1.1" if text.match(/550-5.1.1|This Gmail user does not exist/i)
+      return "5.7.1" if text.match(/5.7.1 Your message.*?was blocked by ROTA DNSBL/i) # AA added
+      return "5.7.2" if text.match(/not have permission to post messages to the group/i)
+      return "5.3.2" if text.match(/Technical details of permanent failure|Too many bad recipients/i) && (text.match(/The recipient server did not accept our requests to connect/i) || text.match(/Connection was dropped by remote host/i) || text.match(/Could not initiate SMTP conversation/i)) # AA added
+      return "4.3.2" if text.match(/Technical details of temporary failure/i) && (text.match(/The recipient server did not accept our requests to connect/i) || text.match(/Connection was dropped by remote host/i) || text.match(/Could not initiate SMTP conversation/i)) # AA added
+      return "5.0.0" if text.match(/Delivery to the following recipient failed permanently/i) # AA added
+      return '5.2.3' if text.match(/account closed|account has been disabled or discontinued|mailbox not found|prohibited by administrator|access denied|account does not exist/i)
+    end
   end
 end

data/lib/schleuder/plugins/attach_listkey.rb

@@ -1,16 +1,12 @@
 module Schleuder
   module ListPlugins
     def self.attach_listkey(arguments, list, mail)
-      filename = "#{list.fingerprint}.pgpkey"
-      # "Mail" only really converts to multipart if the content-type is blank.
-      mail.content_type = nil
-      mail.add_file({
-        filename: filename,
-        content: list.export_key
-      })
-      mail.attachments[filename].content_type = 'application/pgp-keys'
-      mail.attachments[filename].content_description = "OpenPGP public key of #{list.email}"
-      mail.attachments[filename].content_disposition = "attachment; filename=#{filename}"
+      new_part = Mail::Part.new
+      new_part.body = list.export_key
+      new_part.content_type = 'application/pgp-keys'
+      new_part.content_description = "OpenPGP public key of #{list.email}"
+      new_part.content_disposition = "attachment; filename=#{list.fingerprint}.pgpkey"
+      mail.add_part new_part
       nil
     end
   end

data/lib/schleuder/plugins/key_management.rb

@@ -104,35 +104,18 @@ module Schleuder
     # helper methods
     private
 
-    def self.is_armored_key?(material)
-      return false unless /^-----BEGIN PGP PUBLIC KEY BLOCK-----$/ =~ material
-      return false unless /^-----END PGP PUBLIC KEY BLOCK-----$/ =~ material
-
-      lines = material.split("\n").reject(&:empty?)
-      # remove header
-      lines.shift
-      # remove tail
-      lines.pop
-      # verify the rest
-      # TODO: verify length except for lasts lines?
-      # headers according to https://tools.ietf.org/html/rfc4880#section-6.2
-      lines.map do |line|
-        /\A((comment|version|messageid|hash|charset):.*|[0-9a-z\/=+]+)\Z/i =~ line
-      end.all?
-    end
-
     def self.import_keys_from_attachments(list, mail)
       mail.attachments.map do |attachment|
         material = attachment.body.to_s
 
-        list.import_key(material) if self.is_armored_key?(material)
+        list.import_key(material)
       end
     end
 
     def self.import_key_from_body(list, mail)
       key_material = mail.first_plaintext_part.body.to_s
 
-      list.import_key(key_material) if self.is_armored_key?(key_material)
+      list.import_key(key_material)
     end
   end
 end

data/lib/schleuder/plugins/resend.rb

@@ -56,6 +56,9 @@ module Schleuder
 
       # Only continue if all recipients are still here.
       if recip_map.size < arguments.size
+        recip_map.keys.each do |aborted_sender|
+          mail.add_pseudoheader(:error, I18n.t("plugins.resend.aborted", email: aborted_sender))
+        end
         return
       end
 
@@ -117,22 +120,22 @@ module Schleuder
       Array(recipients).inject({}) do |hash, email|
         keys = mail.list.keys(email)
         # Exclude unusable keys.
-        keys.select! { |key| key.usable_for?(:encrypt) }
-        case keys.size
+        usable_keys = keys.select { |key| key.usable_for?(:encrypt) }
+        case usable_keys.size
         when 1
-          hash[email] = keys.first
+          hash[email] = usable_keys.first
         when 0
           if encrypted_only
             # Don't add the email to the result to exclude it from the
             # recipients.
-            add_keys_error(mail, email, keys.size)
+            add_resend_msg(mail, email, :error, 'not_resent_no_keys', usable_keys.size, keys.size)
           else
             hash[email] = ''
           end
         else
           # Always report this situation, regardless of sending or not. It's
           # bad and should be fixed.
-          add_keys_error(mail, email, keys.size)
+          add_resend_msg(mail, email, :notice, 'not_resent_encrypted_no_keys', usable_keys.size, keys.size)
           if ! encrypted_only
             hash[email] = ''
           end
@@ -152,8 +155,8 @@ module Schleuder
       gpg_opts
     end
 
-    def self.add_keys_error(mail, email, keys_size)
-      mail.add_pseudoheader(:error, I18n.t("plugins.resend.not_resent_no_keys", email: email, num_keys: keys_size))
+    def self.add_resend_msg(mail, email, severity, msg, usable_keys_size, all_keys_size)
+      mail.add_pseudoheader(severity, I18n.t("plugins.resend.#{msg}", email: email, usable_keys: usable_keys_size, all_keys: all_keys_size))
     end
 
     def self.add_error_header(mail, recipients_map)
@@ -163,15 +166,15 @@ module Schleuder
     def self.add_resent_headers(mail, recipients_map, to_or_cc, sent_encrypted)
       if sent_encrypted
         prefix = I18n.t('plugins.resend.encrypted_to')
-        str = recipients_map.map do |email, key|
+        str = "\n" + recipients_map.map do |email, key|
           "#{email} (#{key.fingerprint})"
-        end.join(', ')
+        end.join(",\n")
       else
         prefix = I18n.t('plugins.resend.unencrypted_to')
-        str = recipients_map.keys.join(', ')
+        str = ' ' + recipients_map.keys.join(", ")
       end
       headername = resent_header_name(to_or_cc)
-      mail.add_pseudoheader(headername, "#{prefix} #{str}")
+      mail.add_pseudoheader(headername, "#{prefix}#{str}")
     end
 
     def self.resent_header_name(to_or_cc)

data/lib/schleuder/runner.rb

@@ -5,15 +5,43 @@ module Schleuder
       return error if error
 
       logger.info "Parsing incoming email."
+
+      # is it valid utf-8?
+      msg_scrubbed = false
+      unless msg.valid_encoding?
+        logger.warn "Converting message due to invalid characters"
+        detection = CharlockHolmes::EncodingDetector.detect(msg)
+        begin
+          msg = CharlockHolmes::Converter.convert(msg, detection[:encoding], 'UTF-8')
+        rescue ArgumentError
+          # it looks like even icu wasn't able to convert
+          # so we scrub the invalid characters to be able to
+          # at least parse the message somehow. Though this might
+          # result in data loss.
+          logger.warn "Scrubbing message due to invalid characters"
+          msg = msg.scrub
+          msg_scrubbed = true
+        end
+      end
+
       @mail = Mail.create_message_to_list(msg, recipient, list)
 
+      if msg_scrubbed
+        @mail.add_pseudoheader(:note, I18n.t("pseudoheaders.scrubbed_message"))
+      end
+
       error = run_filters('pre')
       return error if error
 
       begin
         # This decrypts, verifies, etc.
         @mail = @mail.setup
-      rescue GPGME::Error::DecryptFailed
+
+      rescue GPGME::Error::BadPassphrase,
+             GPGME::Error::DecryptFailed,
+             GPGME::Error::NoData,
+             GPGME::Error::NoSecretKey
+
         logger.warn "Decryption of incoming message failed."
         return Errors::DecryptionFailed.new(list)
       end
@@ -46,7 +74,7 @@ module Schleuder
       # Subscriptions
       logger.debug "Creating clean copy of message"
       copy = @mail.clean_copy(list.headers_to_meta.any?)
-      list.send_to_subscriptions(copy)
+      list.send_to_subscriptions(copy, @mail)
       nil
     end
 

data/lib/schleuder/subscription.rb

@@ -10,6 +10,10 @@ module Schleuder
     validates :fingerprint, allow_blank: true, fingerprint: true
     validates :delivery_enabled, :admin, boolean: true
 
+    before_validation {
+      self.email = Mail::Address.new(self.email).address
+    }
+
     default_scope { order(:email) }
 
     scope :without_fingerprint, -> { where(fingerprint: [nil,'']) }
@@ -37,10 +41,10 @@ module Schleuder
       list.keys("0x#{self.fingerprint}").first
     end
 
-    def send_mail(mail)
+    def send_mail(mail, incoming_mail=nil)
       list.logger.debug "Preparing sending to #{self.inspect}"
 
-      mail = ensure_headers(mail)
+      mail = ensure_headers(mail, incoming_mail)
       gpg_opts = self.list.gpg_sign_options
 
       if self.key.blank?
@@ -66,9 +70,28 @@ module Schleuder
       mail.deliver
     end
 
-    def ensure_headers(mail)
+    def ensure_headers(mail, incoming_mail=nil)
       mail.to = self.email
-      mail.from = self.list.email
+      
+      if self.list.set_reply_to_to_sender? && ! incoming_mail.nil?
+        # If the option "set_reply_to_to_sender" is set to true, we will set the reply-to header 
+        # to the reply-to header given by the original email. If no reply-to header exists in the original email,
+        # the original senders email will be used as reply-to.
+        if ! incoming_mail.reply_to.nil?
+          mail.reply_to = incoming_mail.reply_to
+        else
+          mail.reply_to = incoming_mail.from
+        end
+      end
+
+      if self.list.munge_from? && ! incoming_mail.nil? 
+        # If the option "munge_from" is set to true, we will add the original senders' from-header to ours.
+        # We munge the from-header to avoid issues with DMARC.
+        mail.from = I18n.t("header_munging", from: incoming_mail.from.first, list: self.list.email, list_address: self.list.email)
+      else
+        mail.from = self.list.email
+      end
+
       mail.sender = self.list.bounce_address
       mail
     end

data/lib/schleuder/version.rb

@@ -1,3 +1,3 @@
 module Schleuder
-  VERSION = '3.4.1'
+  VERSION = '3.6.0'
 end

data/locales/de.yml

@@ -121,7 +121,9 @@ de:
         Oder, um einen Schlüssel per HTTP von einem Server zu laden:
         X-FETCH-KEY: https://example.org/keys/mykey.asc
     resend:
-      not_resent_no_keys: Resending an <%{email}> fehlgeschlagen (%{num_keys} Schlüssel gefunden und unverschlüsseltes Senden verboten).
+      not_resent_no_keys: Resending an <%{email}> fehlgeschlagen (%{all_keys} Schlüssel gefunden, davon %{usable_keys} nutzbar. Unverschlüsseltes Senden verboten).
+      not_resent_encrypted_no_keys: Verschlüsseltes Resending an <%{email}> fehlgeschlagen (%{all_keys} Schlüssel gefunden, davon %{usable_keys} nutzbar).
+      aborted: Resending an <%{email}> abgebrochen aufgrund anderer Probleme.
       encrypted_to: Verschlüsselt an
       unencrypted_to: Unverschlüsselt an
       invalid_recipient: "Ungültige Emailadresse für resend: %{address}"
@@ -250,6 +252,7 @@ de:
   fetch_key:
     invalid_input: "Ungültige Angabe. Gültig sind: URLs, OpenPGP-Fingerabdrücke, oder Emailadressen."
   pseudoheaders:
+    scrubbed_message: Diese Email enthielt ungültige Zeichen, die aus Verarbeitungsgründen möglicherweise entfernt wurden.
     stripped_html_from_multialt: Diese Email enthielt einen alternativen HTML-Teil, der PGP-Daten beinhaltete. Der HTML-Teil wurde entfernt, um die Email sauberer analysieren zu können.
     stripped_html_from_multialt_with_keywords: Diese Email enthielt Schlüsselwörter und einen alternativen HTML-Teil. Der HTML-Teil wurde entfernt, um zu verhindern dass diese Schlüsselwörter Aussenstehenden bekannt werden.
   signature_states:
@@ -258,6 +261,7 @@ de:
   encryption_states:
     encrypted: "Verschlüsselt"
     unencrypted: "Unverschlüsselt"
+  header_munging: "%{from} über %{list} <%{list_address}>"
 
   activerecord:
     errors:

data/locales/en.yml

@@ -125,7 +125,9 @@ en:
         Or, to fetch a key keys by URL:
         X-FETCH-KEY: https://example.org/keys/mykey.asc
     resend:
-      not_resent_no_keys: Resending to <%{email}> failed (%{num_keys} keys found and unencrypted sending disallowed).
+      not_resent_no_keys: Resending to <%{email}> failed (%{all_keys} keys found, of which %{usable_keys} can be used. Unencrypted sending not allowed).
+      not_resent_encrypted_no_keys: Resending as encrypted email to <%{email}> failed (%{all_keys} keys found, of which %{usable_keys} can be used).
+      aborted: Resending to <%{email}> aborted due to other errors.
       encrypted_to: Encrypted to
       unencrypted_to: Unencrypted to
       invalid_recipient: "Invalid email-address for resending: %{address}"
@@ -254,6 +256,7 @@ en:
   fetch_key:
     invalid_input: "Invalid input. Allowed are: URLs, OpenPGP-fingerprints, or email-addresses."
   pseudoheaders:
+    scrubbed_message: This message included invalid characters, which might have been removed to be able to process the message properly.
     stripped_html_from_multialt: This message included an alternating HTML-part that contained PGP-data. The HTML-part was removed to enable parsing the message more properly.
     stripped_html_from_multialt_with_keywords: This message included keywords and an alternating HTML-part. The HTML-part was removed to prevent the disclosure of these keywords to third parties.
   signature_states:
@@ -262,6 +265,7 @@ en:
   encryption_states:
     encrypted: "Encrypted"
     unencrypted: "Unencrypted"
+  header_munging: "%{from} via %{list} <%{list_address}>"
 
   activerecord:
     errors:

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: schleuder
 version: !ruby/object:Gem::Version
-  version: 3.4.1
+  version: 3.6.0
 platform: ruby
 authors:
 - schleuder dev team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-16 00:00:00.000000000 Z
+date: 2021-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gpgme
@@ -19,7 +19,7 @@ dependencies:
         version: '2.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.13
+        version: 2.0.19
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
         version: '2.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.13
+        version: 2.0.19
 - !ruby/object:Gem::Dependency
   name: mail
   requirement: !ruby/object:Gem::Requirement
@@ -51,9 +51,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.3'
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.3.3
+        version: 0.4.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -61,9 +61,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.3'
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.3.3
+        version: 0.4.3
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -176,6 +176,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
+- !ruby/object:Gem::Dependency
+  name: charlock_holmes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.6
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -249,7 +263,7 @@ dependencies:
 description: |-
   Schleuder is a group's email-gateway: subscribers can exchange encrypted emails among themselves, receive emails from non-subscribers and send emails to non-subscribers via the list.
 
-  (Please note: For some platforms there's a better way of installing Schleuder than `gem install`. See <https://schleuder.org/docs/#installation> for details.)
+  (Please note: For some platforms there's a better way of installing Schleuder than `gem install`. See <https://schleuder.org/schleuder/docs/server-admins.html#installation> for details.)
 email: team@schleuder.org
 executables:
 - schleuder
@@ -275,6 +289,9 @@ files:
 - db/migrate/20160501172700_fix_headers_to_meta_defaults.rb
 - db/migrate/20170713215059_add_internal_footer_to_list.rb
 - db/migrate/20180110203100_add_sig_enc_to_headers_to_meta_defaults.rb
+- db/migrate/20180723173900_add_deliver_selfsent_to_list.rb
+- db/migrate/20190906194820_add_autocrypt_header_to_list.rb
+- db/migrate/20200118170110_add_set_reply_to_to_sender_and_munge_from.rb
 - db/schema.rb
 - etc/init.d/schleuder-api-daemon
 - etc/list-defaults.yml
@@ -391,7 +408,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: "[none]"
+rubyforge_project: 
 rubygems_version: 2.7.6.2
 signing_key: 
 specification_version: 4