schleuder 3.5.0 → 4.0.0

data/etc/schleuder-weekly-key-maintenance.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Schleuder weekly key maintenance
+
+[Timer]
+OnCalendar=weekly
+Persistent=true
+
+[Install]
+WantedBy=timers.target

data/etc/schleuder.yml

@@ -4,8 +4,8 @@ lists_dir: /var/lib/schleuder/lists
 # Where to write list-logs. The actual log-file will be <lists_logs_base_dir>/<hostname>/<listname>/list.log.
 listlogs_dir: /var/lib/schleuder/lists
 
-# Schleuder reads plugins also from this directory.
-plugins_dir: /etc/schleuder/plugins
+# Schleuder looks for additional, custom keyword-handlers in this directory.
+keyword_handlers_dir: /usr/local/lib/schleuder/keyword_handlers
 
 # Schleuder reads filters also from this directory path,
 # in the specific pre_decryption or post_decryption subdirectory.
@@ -22,7 +22,7 @@ filters_dir: /usr/local/lib/schleuder/filters
 log_level: warn
 
 # Which keyserver to refresh keys from (used by `schleuder refresh_keys`, meant
-# to be run from cron weekly).
+# to be run from cron or systemd weekly).
 # If you have gnupg 2.1, we strongly suggest to use a hkps-keyserver:
 #keyserver: hkps://hkps.pool.sks-keyservers.net
 # If you have gnupg 2.1 and TOR running locally, use a onion-keyserver:

data/lib/schleuder-api-daemon/helpers/schleuder-api-daemon-helper.rb

@@ -23,7 +23,7 @@ module SchleuderApiDaemonHelper
         if params[:list_id].present?
           id_or_email = params[:list_id]
         else
-          client_error "Parameter list_id is required"
+          client_error 'Parameter list_id is required'
         end
       end
       if is_an_integer?(id_or_email)
@@ -41,7 +41,7 @@ module SchleuderApiDaemonHelper
       else
         # Email
         if params[:list_id].blank?
-          client_error "Parameter list_id is required when using email as identifier for subscriptions."
+          client_error 'Parameter list_id is required when using email as identifier for subscriptions.'
         else
           sub = list.subscriptions.where(email: id_or_email).first
         end
@@ -103,7 +103,7 @@ module SchleuderApiDaemonHelper
         expiry: key.expires,
         generated_at: key.generated_at,
         primary_uid: key.primary_uid.uid,
-        oneline: key.oneline,
+        summary: key.summary,
         trust_issues: key.usability_issue
       }
       if include_keydata

data/lib/schleuder-api-daemon/routes/subscription.rb

@@ -3,9 +3,9 @@ class SchleuderApiDaemon < Sinatra::Base
 
   namespace '/subscriptions' do
     get '.json' do
-      filterkeys = Subscription.configurable_attributes + [:list_id, :email]
+      filterkeys = Subscription.configurable_attributes + ['list_id', 'email']
       filter = params.select do |param|
-        filterkeys.include?(param.to_sym)
+        filterkeys.include?(param)
       end
 
       logger.debug "Subscription filter: #{filter.inspect}"
@@ -43,7 +43,7 @@ class SchleuderApiDaemon < Sinatra::Base
           client_error(sub, 422)
         end
       rescue ActiveRecord::RecordNotUnique
-        logger.error "Already subscribed"
+        logger.error 'Already subscribed'
         status 422
         json errors: {email: ['is already subscribed']}
       end
@@ -70,7 +70,7 @@ class SchleuderApiDaemon < Sinatra::Base
       # For an already existing subscription, only update fingerprint if a
       # new one has been selected from the upload.
       if fingerprint.present?
-        args["fingerprint"] = fingerprint
+        args['fingerprint'] = fingerprint
       end
       if sub.update(args)
         200

data/lib/schleuder.rb

@@ -1,4 +1,4 @@
-# default to ASCII-8BIT encoding as early as possible for external
+# default to UTF-8 encoding as early as possible for external
 # data.
 #
 # this should ensure we are able to parse most incoming
@@ -17,6 +17,8 @@ require 'open3'
 # Require mandatory libs. The database-layer-lib is required below.
 require 'mail-gpg'
 require 'active_record'
+require 'active_support'
+require 'active_support/core_ext/string'
 
 # An extra from mail-gpg
 require 'hkp'
@@ -48,10 +50,9 @@ require 'schleuder/version'
 require 'schleuder/logger_notifications'
 require 'schleuder/logger'
 require 'schleuder/listlogger'
-require 'schleuder/plugin_runners/base'
-require 'schleuder/plugin_runners/list_plugins_runner'
-require 'schleuder/plugin_runners/request_plugins_runner'
-Dir["#{libdir}/schleuder/plugins/*.rb"].each do |file|
+require 'schleuder/keyword_handlers_runner'
+require 'schleuder/keyword_handlers/base'
+Dir["#{libdir}/schleuder/keyword_handlers/*.rb"].each do |file|
   require file
 end
 require 'schleuder/filters_runner'
@@ -64,13 +65,10 @@ require 'schleuder/list_builder'
 require 'schleuder/subscription'
 
 # Setup
-ENV["SCHLEUDER_CONFIG"] ||= '/etc/schleuder/schleuder.yml'
-ENV["SCHLEUDER_LIST_DEFAULTS"] ||= '/etc/schleuder/list-defaults.yml'
-ENV["SCHLEUDER_ENV"] ||= 'production'
-ENV["SCHLEUDER_ROOT"] = rootdir.to_s
-# Ensure that gnupg never-ever tries to ask for a passphrase.
-ENV["GPG_TTY"] = "/nonexistant-#{rand}"
-ENV["DISPLAY"] = nil
+ENV['SCHLEUDER_CONFIG'] ||= '/etc/schleuder/schleuder.yml'
+ENV['SCHLEUDER_LIST_DEFAULTS'] ||= '/etc/schleuder/list-defaults.yml'
+ENV['SCHLEUDER_ENV'] ||= 'production'
+ENV['SCHLEUDER_ROOT'] = rootdir.to_s
 
 GPGME::Ctx.set_gpg_path_from_env
 GPGME::Ctx.check_gpg_version

data/lib/schleuder/cli.rb

@@ -5,11 +5,13 @@ require 'charlock_holmes'
 
 require_relative '../schleuder'
 require 'schleuder/cli/subcommand_fix'
+require 'schleuder/cli/cli_helper'
 require 'schleuder/cli/schleuder_cert_manager'
 require 'schleuder/cli/cert'
 
 module Schleuder
   class Cli < Thor
+    include CliHelper
 
     register(Cert,
              'cert',
@@ -51,7 +53,7 @@ module Schleuder
       exit 1
     end
 
-    desc 'check_keys', 'Check all lists for unusable or expiring keys and send the results to the list-admins. (This is supposed to be run from cron weekly.)'
+    desc 'check_keys', 'Check all lists for unusable or expiring keys and send the results to the list-admins. (This is supposed to be run from cron or systemd weekly.)'
     def check_keys
       List.all.each do |list|
         I18n.locale = list.language
@@ -66,30 +68,26 @@ module Schleuder
       permission_notice
     end
 
-    desc 'refresh_keys [list1@example.com]', "Refresh all keys of all list from the keyservers sequentially (one by one or on the passed list). (This is supposed to be run from cron weekly.)"
+    desc 'refresh_keys [list1@example.com]', 'Refresh all keys of all list from the keyservers sequentially (one by one or on the passed list). (This is supposed to be run from cron or systemd weekly.)'
     def refresh_keys(list=nil)
       GPGME::Ctx.send_notice_if_gpg_does_not_know_import_filter
-      work_on_lists(:refresh_keys,list)
+      work_on_lists(:refresh_keys, list)
       permission_notice
     end
 
-    desc 'pin_keys [list1@example.com]', "Find keys for subscriptions without a pinned key and try to pin a certain key (one by one or based on the passed list)."
-    def pin_keys(list=nil)
-      work_on_lists(:pin_keys,list)
-    end
 
-    desc 'install', "Set-up or update Schleuder environment (create folders, copy files, fill the database)."
+    desc 'install', 'Set-up or update Schleuder environment (create folders, copy files, fill the database).'
     def install
       config_dir = Pathname.new(ENV['SCHLEUDER_CONFIG']).dirname
       root_dir = Pathname.new(ENV['SCHLEUDER_ROOT'])
 
       # Check if lists_dir contains v2-data.
       if Dir.glob("#{Conf.lists_dir}/*/*/members.conf").size > 0
-        msg = "Lists directory #{Conf.lists_dir} appears to contain data from a Schleuder version 2.x installation.\nPlease move it out of the way or configure a different `lists_dir` in `#{ENV['SCHLEUDER_CONFIG']}`.\nTo migrate lists from Schleuder v2 to Schleuder v3 please use `schleuder migrate_v2_list` after the installation succeeded."
+        msg = "Lists directory #{Conf.lists_dir} appears to contain data from a Schleuder version 2.x installation.\nPlease remove this data and retry the installation. Schleuder version 4 doesn't support migrating these old lists, in case you need to, please install Schleuder version 3 first."
         fatal msg, 2
       end
 
-      [Conf.lists_dir, Conf.listlogs_dir, config_dir].each do |dir|
+      [Conf.keyword_handlers_dir, Conf.lists_dir, Conf.listlogs_dir, config_dir].each do |dir|
         dir = Pathname.new(dir)
         if ! dir.exist?
           begin
@@ -101,7 +99,7 @@ module Schleuder
         end
       end
 
-      Pathname.glob(root_dir.join("etc").join("*.yml")).each do |file|
+      Pathname.glob(root_dir.join('etc').join('*.yml')).each do |file|
         target = config_dir.join(file.basename)
         if ! target.exist?
           if target.dirname.writable?
@@ -131,184 +129,7 @@ module Schleuder
       fatal exc.message
     end
 
-    desc 'migrate-v2-list /path/to/listdir', 'Migrate list from v2.2 to v3.'
-    def migrate_v2_list(path)
-      dir = Pathname.new(path)
-      if ! dir.readable? || ! dir.directory?
-        fatal "Not a readable directory: `#{path}`."
-      end
-
-      %w[list.conf members.conf pubring.gpg].each do |file|
-        if ! (dir + file).exist?
-          fatal "Not a complete schleuder v2.2 listdir: missing #{file}"
-        end
-      end
-
-      conf = YAML.load(File.read(dir + 'list.conf'))
-      if conf.nil? || conf.empty?
-        fatal "list.conf is blank"
-      end
-      listname = conf['myaddr']
-      if listname.nil? || listname.empty?
-        fatal "myaddr is blank in list.conf"
-      end
-
-      # Identify list-fingerprint.
-      ENV['GNUPGHOME'] = dir.to_s
-      listkey = GPGME::Key.find(:public, "<#{listname}>").first
-      if listkey.nil?
-        fatal "Failed to identify the list's OpenPGP-key!"
-      end
-
-      # Create list.
-      begin
-        list, messages = Schleuder::ListBuilder.new({email: listname, fingerprint: listkey.fingerprint}).run
-      rescue => exc
-        fatal exc
-      end
-      if messages
-        fatal messages.values.join(" - ")
-      elsif list.errors.any?
-        fatal list.errors.full_messages.join(" - ")
-      end
-
-      # Import keys
-      list.import_key(File.read(dir + 'pubring.gpg'))
-      list.import_key(File.read(dir + 'secring.gpg'))
-
-      # Clear passphrase of imported list-key.
-      output = list.key.clearpassphrase(conf['gpg_password'])
-      if output.present?
-        fatal "while clearing passphrase of list-key: #{output.inspect}"
-      end
-
-      # Set list-options.
-      List.configurable_attributes.each do |option|
-        option = option.to_s
-        if conf.keys.include?(option)
-          value = case option
-                  when /^keywords_/
-                    filter_keywords(conf[option])
-                  when 'log_level'
-                    conf[option].to_s.downcase
-                  else
-                    conf[option]
-                  end
-          list.set_attribute(option, value)
-        end
-      end
-
-      # Set changed options.
-      changed_options = {
-        'prefix' => 'subject_prefix',
-        'prefix_in' => 'subject_prefix_in',
-        'prefix_out' => 'subject_prefix_out',
-        'dump_incoming_mail' => 'forward_all_incoming_to_admins',
-        'receive_from_member_emailaddresses_only' => 'receive_from_subscribed_emailaddresses_only',
-        'bounces_notify_admin' => 'bounces_notify_admins',
-        'max_message_size' => 'max_message_size_kb'
-      }
-
-      changed_options.each do |old, new|
-        if conf.keys.include?(old)
-          list.set_attribute(new, conf[old])
-        end
-      end
-      list.save!
-
-      # Subscribe members
-      members = YAML.load(File.read(dir + 'members.conf'))
-      members.uniq!{|m| m['email'] }
-      members.each do |member|
-        fingerprint = find_fingerprint(member, list)
-        list.subscribe(member['email'], fingerprint)
-      end
-
-      # Subscribe or flag admins
-      conf['admins'].each do |member|
-        sub = list.subscriptions.where(email: member['email']).first
-        if sub
-          sub.admin = true
-          sub.save!
-        else
-          adminfpr = find_fingerprint(member, list)
-          # if we didn't find an already imported  subscription for the admin
-          # address, it wasn't a member, so we don't enable delivery for it
-          list.subscribe(member['email'], adminfpr, true, false)
-        end
-      end
-
-      # Notify of removed options
-      say "Please note: the following options have been *removed*:
-* `default_mime` for lists (we only support pgp/mime for now),
-* `archive` for lists,
-* `gpg_passphrase` for lists,
-* `log_file`, `log_io`, `log_syslog` for lists (we only log to
-         syslog (before list-creation) and a file (after it) for now),
-* `mime` for subscriptions/members (we only support pgp/mime for now),
-* `send_encrypted_only` for members/subscriptions.
-
-If you really miss any of them please tell us.
-
-Please also note that the following keywords have been renamed:
-* list-members  => list-subscriptions
-* add-member    => subscribe
-* delete-member => unsubscribe
-
-Please notify the users and admins of this list of these changes.
-"
-
-      say "\nList #{listname} migrated to schleuder v3."
-      if messages.present?
-        say messages.gsub(' // ', "\n")
-      end
-      permission_notice
-    rescue => exc
-      fatal "#{exc}\n#{exc.backtrace.first}"
-    end
-
     no_commands do
-      def fatal(msg, exitcode=1)
-        error("Error: #{msg}")
-        exit exitcode
-      end
-
-      KEYWORDS = {
-        'add-member' => 'subscribe',
-        'delete-member' => 'unsubscribe',
-        'list-members' => 'list-subscriptions',
-        'subscribe' => 'subscribe',
-        'unsubscribe' => 'unsubscribe',
-        'list-subscriptions' => 'list-subscriptions',
-        'set-finterprint' => 'set-fingerprint',
-        'add-key' => 'add-key',
-        'delete-key' => 'delete-key',
-        'list-keys' => 'list-keys',
-        'get-key' => 'get-key',
-        'fetch-key' => 'fetch-key'
-      }
-
-      def filter_keywords(value)
-        Array(value).map do |keyword|
-          KEYWORDS[keyword.downcase]
-        end.compact
-      end
-
-      def find_fingerprint(member, list)
-        email = member['email']
-        fingerprint = member['key_fingerprint']
-        if fingerprint.present?
-          return fingerprint
-        end
-
-        key = list.distinct_key(email)
-        if key
-          return key.fingerprint
-        else
-          return nil
-        end
-      end
-
       def shellexec(cmd)
         result = `#{cmd} 2>&1`
         if $?.exitstatus > 0

data/lib/schleuder/cli/cert.rb

@@ -8,9 +8,9 @@ module Schleuder
       cert = Conf.api['tls_cert_file']
       fingerprint = SchleuderCertManager.generate('schleuder', key, cert)
       puts "Fingerprint of generated certificate: #{fingerprint}"
-      puts "Have this fingerprint included into the configuration-file of all clients that want to connect to your Schleuder API."
+      puts 'Have this fingerprint included into the configuration-file of all clients that want to connect to your Schleuder API.'
       if Process.euid == 0
-        puts "! Warning: this process was run as root — please make sure the above files are accessible by the user that is running `schleuder-api-daemon`."
+        puts '! Warning: this process was run as root — please make sure the above files are accessible by the user that is running `schleuder-api-daemon`.'
       end
     end
 

data/lib/schleuder/cli/cli_helper.rb

@@ -0,0 +1,14 @@
+module Schleuder
+  module CliHelper
+    def self.included(base)
+      base.no_commands do
+
+        def fatal(msg, exitcode=1)
+          error("Error: #{msg}")
+          exit exitcode
+        end
+
+      end
+    end
+  end
+end

data/lib/schleuder/cli/schleuder_cert_manager.rb

@@ -22,11 +22,11 @@ class SchleuderCertManager
     ef.subject_certificate = cert
     ef.issuer_certificate = cert
     cert.extensions = [
-      ef.create_extension("basicConstraints","CA:TRUE", true),
-      ef.create_extension("subjectKeyIdentifier", "hash"),
+      ef.create_extension('basicConstraints', 'CA:TRUE', true),
+      ef.create_extension('subjectKeyIdentifier', 'hash'),
     ]
-    cert.add_extension ef.create_extension("authorityKeyIdentifier",
-                                           "keyid:always,issuer:always")
+    cert.add_extension ef.create_extension('authorityKeyIdentifier',
+                                           'keyid:always,issuer:always')
 
     cert.sign key, OpenSSL::Digest::SHA256.new
 

data/lib/schleuder/conf.rb

@@ -4,14 +4,20 @@ module Schleuder
   class Conf
     include Singleton
 
-    EMAIL_REGEXP = /\A.+@[[:alnum:]_.-]+\z/i
+    # since the regexp got only included into stdlib 2.2
+    # TODO: remove once 2.1 support dropped
+    if RUBY_VERSION < '2.2'
+      EMAIL_REGEXP = /\A[a-zA-Z0-9.!\#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*\z/
+    else
+      EMAIL_REGEXP = URI::MailTo::EMAIL_REGEXP
+    end
     # TODO: drop v3 keys and only accept length of 40
     FINGERPRINT_REGEXP = /\A(0x)?[a-f0-9]{32}([a-f0-9]{8})?\z/i
 
     DEFAULTS = {
       'lists_dir' => '/var/lib/schleuder/lists',
       'listlogs_dir' => '/var/lib/schleuder/lists',
-      'plugins_dir' => '/etc/schleuder/plugins',
+      'keyword_handlers_dir' => '/usr/local/lib/schleuder/keyword_handlers',
       'filters_dir' => '/usr/local/lib/schleuder/filters',
       'log_level' => 'warn',
       'superadmin' => 'root@localhost',
@@ -56,8 +62,8 @@ module Schleuder
       instance.config['listlogs_dir']
     end
 
-    def self.plugins_dir
-      instance.config['plugins_dir']
+    def self.keyword_handlers_dir
+      instance.config['keyword_handlers_dir']
     end
 
     def self.filters_dir