sup 1.0 → 1.2

data/test/fixtures/non-ascii-header-in-nested-message.eml

@@ -0,0 +1,36 @@
+Return-Path: <spammer@example.com>
+From: SPAM � <spammer@example.com>
+To: <recipient@example.invalid>
+Subject: spam � spam
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="----------=_4F506AC2.EE281DC4"
+Message-Id: <20120302063755.0FE2122017@a.a.a.a>
+Date: Fri,  2 Mar 2012 07:37:55 +0100 (CET)
+
+This is a multi-part message in MIME format.
+
+------------=_4F506AC2.EE281DC4
+Content-Type: text/plain; charset=iso-8859-1
+Content-Disposition: inline
+Content-Transfer-Encoding: 8bit
+
+Spam detection software, running on the system "a.a.a.a.a.", has
+identified this incoming email as possible spam.  The original message
+has been attached to this so you can view it (if it isn't spam) or label
+similar future email.
+
+
+------------=_4F506AC2.EE281DC4
+Content-Type: message/rfc822; x-spam-type=original
+Content-Description: original message before SpamAssassin
+Content-Disposition: attachment
+Content-Transfer-Encoding: 8bit
+
+From: SPAM � <spammer@example.com>
+To: <enclosed@example.invalid>
+Subject: spam � spam
+
+This is a spam.
+
+------------=_4F506AC2.EE281DC4--
+

data/test/fixtures/non-ascii-header.eml

@@ -0,0 +1,8 @@
+Return-Path: <spammer@example.com>
+From: SPAM � <spammer@example.com>
+To: <a@b.c>
+Subject: spam � spam
+Message-Id: <20120302063755.0FE2122017@a.a.a.a>
+Date: Fri,  2 Mar 2012 07:37:55 +0100 (CET)
+
+https://github.com/sup-heliotrope/sup/issues/205

data/test/fixtures/rfc2047-header-encoding.eml

@@ -0,0 +1,15 @@
+From: test@example.invalid
+To: test@example.invalid
+Date: Sun, 19 Jul 2020 17:03:56 +1000
+Subject:
+ =?US-ASCII?q?Hans Martin Djupvik?= =?ISO-8859-1?q?,_Ingrid_B=F8?=
+ =?KOI8-R?b?LCDp0snOwSDzycTP0s/XwQ?=
+ =?UTF-16?b?//4sACAASgBlAHMAcABlAHIAIABCAGUAcgBnAA?=
+ =?UTF-7?b?LCBGcmlkYSBFbmcrQVBnLQ?=
+ bad: =?UTF16?q?badcharsetname?= =?US-ASCII?b?/w?=
+ =?UTF-7?Q?=41=6D=65=72=69=63=61=E2=80=99=73?=
+
+The subject header contains various RFC2047 encoded words.
+For completeness we test both base64 and quoted-printable, and some
+ASCII-incompatible encodings.
+We also include some bogus words which cannot be decoded.

data/test/fixtures/text-attachments-with-charset.eml

@@ -42,5 +42,19 @@ Content-Disposition: attachment; filename="bad.txt"
 MIME-Version: 1.0
 
 Embedded=F0garbage
---===============2385509127900810307==--
+
+--===============2385509127900810307==
+Content-Type: text/plain; charset="invalid-test"; name="invalid-charset.txt"
+Content-Transfer-Encoding: quoted-printable
+Content-Disposition: attachment; filename="invalid-charset.txt"
+
+Example invalid charset
+--===============2385509127900810307==
+Content-Type: text/plain; charset="utf-7"
+Content-Transfer-Encoding: 7bit
+MIME-Version: 1.0
+Content-Disposition: attachment; filename="ascii.txt"
+
+This is +Jyg-UTF-7+Jyg-
+--===============2385509127900810307==
 

data/test/fixtures/utf8-header.eml

@@ -0,0 +1,17 @@
+Delivered-To: djc@djc.id.au
+Received: from orpheus.librarything.com (orpheus.librarything.com [74.201.105.9])
+    by djc.id.au (Postfix) with ESMTP id A0CCB20AAB99
+    for <djc@djc.id.au>; Sat, 23 Jan 2021 02:52:15 +1000 (AEST)
+Received: by orpheus.librarything.com (Postfix, from userid 0)
+    id 21B172C20F9; Fri, 22 Jan 2021 11:52:08 -0500 (EST)
+To: djc@djc.id.au
+Subject: LibraryThing: State of the Thing — January
+MIME-Version: 1.0
+Content-type: text/html; charset=iso-8859-1
+From: tim@librarything.com
+Reply-To: tim@librarything.com
+Message-Id: <20210122165208.21B172C20F9@orpheus.librarything.com>
+Date: Fri, 22 Jan 2021 11:52:08 -0500 (EST)
+Return-Path: <root@orpheus.librarything.com>
+
+<p>Some stuff</p>

data/test/integration/test_maildir.rb

@@ -1,7 +1,10 @@
+require "sup"
 require "test_helper"
 
 class TestMaildir < Minitest::Test
 
+  include Redwood
+
   def setup
     @path = Dir.mktmpdir
 

data/test/integration/test_mbox.rb

@@ -1,6 +1,9 @@
+require "sup"
 require "test_helper"
 
-class TestMbox < MiniTest::Test
+class TestMbox < Minitest::Test
+
+  include Redwood
 
   def setup
     @path = Dir.mktmpdir

data/test/integration/test_sup-add.rb

@@ -0,0 +1,83 @@
+class TestSupAdd < Minitest::Test
+
+  def setup
+    @path = Dir.mktmpdir
+  end
+
+  def teardown
+    FileUtils.rm_r @path
+  end
+
+  def test_can_add_maildir_source
+    _out, _err = capture_subprocess_io do
+      assert system({"SUP_BASE" => @path}, "bin/sup-add", "maildir:///some/path")
+    end
+
+    generated_sources_yaml = File.read "#{@path}/sources.yaml"
+    assert_equal <<EOS, generated_sources_yaml
+---
+- !<tag:supmua.org,2006-10-01/Redwood/Maildir>
+  uri: maildir:///some/path
+  usual: true
+  archived: false
+  sync_back: true
+  id: 1
+  labels: []
+EOS
+  end
+
+  def test_fixes_old_tag_uri_syntax
+    File.write "#{@path}/sources.yaml", <<EOS
+---
+- !supmua.org,2006-10-01/Redwood/Maildir
+  uri: maildir:/some/path
+  usual: true
+  archived: false
+  sync_back: true
+  id: 1
+  labels: []
+EOS
+    _out, _err = capture_subprocess_io do
+      assert system({"SUP_BASE" => @path}, "bin/sup-add", "maildir:///other/path")
+    end
+
+    generated_sources_yaml = File.read "#{@path}/sources.yaml"
+    assert_equal <<EOS, generated_sources_yaml
+---
+- !<tag:supmua.org,2006-10-01/Redwood/Maildir>
+  uri: maildir:/some/path
+  usual: true
+  archived: false
+  sync_back: true
+  id: 1
+  labels: []
+- !<tag:supmua.org,2006-10-01/Redwood/Maildir>
+  uri: maildir:///other/path
+  usual: true
+  archived: false
+  sync_back: true
+  id: 2
+  labels: []
+EOS
+  end
+
+  ## https://github.com/sup-heliotrope/sup/issues/545
+  def test_source_with_invalid_uri_chars_in_path
+    _out, _err = capture_subprocess_io do
+      assert system({"SUP_BASE" => @path}, "bin/sup-add", "maildir:~/.mail/gmail/[Gmail]/.All Mail/")
+    end
+
+    generated_sources_yaml = File.read "#{@path}/sources.yaml"
+    assert_equal <<EOS, generated_sources_yaml
+---
+- !<tag:supmua.org,2006-10-01/Redwood/Maildir>
+  uri: maildir:~/.mail/gmail/[Gmail]/.All Mail/
+  usual: true
+  archived: false
+  sync_back: true
+  id: 1
+  labels: []
+EOS
+  end
+
+end

data/test/integration/test_sup-sync-back-maildir.rb

@@ -0,0 +1,40 @@
+require "test_helper"
+
+class TestSupSyncBackMaildir < Minitest::Test
+
+  def setup
+    @path = Dir.mktmpdir
+
+    @maildir = File.join @path, "test_maildir"
+    Dir.mkdir @maildir
+    %w[cur new tmp].each do |subdir|
+      Dir.mkdir (File.join @maildir, subdir)
+    end
+    msg_path = File.join @maildir, "new", "123.hostname"
+    FileUtils.copy_file fixture_path("simple-message.eml"), msg_path
+
+    _out, _err = capture_subprocess_io do
+      assert system({"SUP_BASE" => @path}, "bin/sup-add", "maildir://#{@maildir}")
+      assert system({"SUP_BASE" => @path}, "bin/sup-sync")
+    end
+  end
+
+  def teardown
+    FileUtils.rm_r @path
+  end
+
+  def test_it_syncs_seen_unread_flags
+    _out, _err = capture_subprocess_io do
+      assert system({"SUP_BASE" => @path},
+                    "bin/sup-tweak-labels",
+                    "--all-sources",
+                    "--add=replied",
+                    "--remove=unread")
+      assert system({"SUP_BASE" => @path}, "bin/sup-sync-back-maildir", "--no-confirm")
+    end
+
+    refute File.exist? (File.join @maildir, "new", "123.hostname")
+    assert File.exist? (File.join @maildir, "cur", "123.hostname:2,RS")
+  end
+
+end

data/test/test_crypto.rb

@@ -69,6 +69,24 @@ class TestCryptoManager < Minitest::Test
         end
     end
 
+    def test_sign_nested_parts
+        if CryptoManager.have_crypto? then
+            body = RMail::Message.new
+            body.header["Content-Disposition"] = "inline"
+            body.body = "ABCDEFG"
+            payload = RMail::Message.new
+            payload.header["MIME-Version"] = "1.0"
+            payload.add_part body
+            payload.add_part RMail::Message.make_attachment "attachment", "text/plain", nil, "attachment.txt"
+            signed = CryptoManager.sign @from_email, @to_email, payload
+            ## The result is a multipart/signed containing a multipart/mixed.
+            ## There should be a MIME-Version header on the top-level
+            ## multipart/signed message, but *not* on the enclosed
+            ## multipart/mixed part.
+            assert_equal 1, signed.to_s.scan(/MIME-Version:/).size
+        end
+    end
+
     def test_encrypt
         if CryptoManager.have_crypto? then
             encrypted = CryptoManager.encrypt @from_email, [@to_email], "ABCDEFG"
@@ -116,6 +134,32 @@ class TestCryptoManager < Minitest::Test
             CryptoManager.verify signed.body[0], signed.body[1], true
         end
     end
+
+    def test_verify_nested_parts
+        if CryptoManager.have_crypto?
+            ## Generate a multipart/signed containing a multipart/mixed.
+            ## We will test verifying the generated signature below.
+            ## Importantly, the inner multipart/mixed does *not* have a
+            ## MIME-Version header because it is not a top-level message.
+            payload = RMail::Parser.read <<EOS
+Content-Type: multipart/mixed; boundary="=-1652088224-7794-561531-1825-1-="
+
+
+--=-1652088224-7794-561531-1825-1-=
+Content-Disposition: inline
+
+ABCDEFG
+--=-1652088224-7794-561531-1825-1-=
+Content-Disposition: attachment; filename="attachment.txt"
+Content-Type: text/plain; name="attachment.txt"
+
+attachment
+--=-1652088224-7794-561531-1825-1-=--
+EOS
+            signed = CryptoManager.sign @from_email_ecc, @to_email, payload
+            CryptoManager.verify payload, signed.body[1], true
+        end
+    end
 end
 
 end

data/test/test_header_parsing.rb

@@ -4,16 +4,22 @@ require 'test_helper'
 require 'sup'
 require 'stringio'
 
-include Redwood
-
 class TestMBoxParsing < Minitest::Test
 
+  include Redwood
+
   def setup
     @path = Dir.mktmpdir
     @mbox = File.join(@path, 'test_mbox')
+    @log = StringIO.new
+    Redwood::Logger.add_sink @log
+    Redwood::Logger.remove_sink $stderr
   end
 
   def teardown
+    Redwood::Logger.clear!
+    Redwood::Logger.remove_sink @log
+    Redwood::Logger.add_sink $stderr
     FileUtils.rm_r @path
   end
 
@@ -69,7 +75,7 @@ EOS
 
   def test_blank_lines
     h = Source.parse_raw_email_header StringIO.new("")
-    assert_equal nil, h["message-id"]
+    assert_nil h["message-id"]
   end
 
   def test_empty_headers
@@ -133,6 +139,8 @@ EOS
     assert_equal 61, offset
     offset = l.next_offset 61
     assert_nil offset
+    assert_match(/WARNING: found invalid date in potential mbox split line, not splitting/,
+                 @log.string)
   end
 
   def test_more_from_line_splitting

data/test/test_helper.rb

@@ -2,9 +2,12 @@ require "rubygems" rescue nil
 require 'minitest/autorun'
 require "rr"
 
-def fixture(filename)
+def fixture_path(filename)
+  File.expand_path("../fixtures/#{filename}", __FILE__)
+end
+
+def fixture_contents(filename)
   file = ''
-  path = File.expand_path("../fixtures/#{filename}", __FILE__)
-  File.open(path) { |io| file = io.read }
+  File.open(fixture_path(filename)) { |io| file = io.read }
   file
-end
+end

data/test/test_message.rb

@@ -21,10 +21,8 @@ class TestMessage < Minitest::Test
   end
 
   def test_simple_message
-    message = fixture('simple-message.eml')
-
     source = DummySource.new("sup-test://test_simple_message")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('simple-message.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)
@@ -99,10 +97,8 @@ class TestMessage < Minitest::Test
   end
 
   def test_multipart_message
-    message = fixture('multi-part.eml')
-
     source = DummySource.new("sup-test://test_multipart_message")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('multi-part.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)
@@ -126,10 +122,8 @@ class TestMessage < Minitest::Test
   end
 
   def test_broken_message_1
-    message = fixture('missing-from-to.eml')
-
     source = DummySource.new("sup-test://test_broken_message_1")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('missing-from-to.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)
@@ -150,10 +144,8 @@ class TestMessage < Minitest::Test
   end
 
   def test_broken_message_2
-    message = fixture('no-body.eml')
-
     source = DummySource.new("sup-test://test_broken_message_1")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('no-body.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)
@@ -167,10 +159,8 @@ class TestMessage < Minitest::Test
   end
 
   def test_multipart_message_2
-    message = fixture('multi-part-2.eml')
-
     source = DummySource.new("sup-test://test_multipart_message_2")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('multi-part-2.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)
@@ -178,21 +168,20 @@ class TestMessage < Minitest::Test
 
     chunks = sup_message.load_from_source! # read the message body chunks
 
-    # TODO: Add more asserts
+    assert_equal(1, chunks.length)
+    assert(chunks[0].is_a? Redwood::Chunk::Attachment)
   end
 
   def test_text_attachment_decoding
-    message = fixture('text-attachments-with-charset.eml')
-
     source = DummySource.new("sup-test://test_text_attachment_decoding")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('text-attachments-with-charset.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)
     sup_message.load_from_source!
 
     chunks = sup_message.load_from_source!
-    assert_equal(5, chunks.length)
+    assert_equal(7, chunks.length)
     assert(chunks[0].is_a? Redwood::Chunk::Text)
     ## The first attachment declares charset=us-ascii
     assert(chunks[1].is_a? Redwood::Chunk::Attachment)
@@ -207,13 +196,18 @@ class TestMessage < Minitest::Test
     ## which will be replaced with U+FFFD REPLACEMENT CHARACTER
     assert(chunks[4].is_a? Redwood::Chunk::Attachment)
     assert_equal(["Embedded\ufffdgarbage"], chunks[4].lines)
+    ## The fifth attachment has an invalid charset, which should still
+    ## be handled gracefully
+    assert(chunks[5].is_a? Redwood::Chunk::Attachment)
+    assert_equal(["Example invalid charset"], chunks[5].lines)
+    ## The sixth attachment is UTF-7 encoded
+    assert(chunks[6].is_a? Redwood::Chunk::Attachment)
+    assert_equal(["This is ✨UTF-7✨"], chunks[6].lines)
   end
 
   def test_mailing_list_header
-    message = fixture('mailing-list-header.eml')
-
     source = DummySource.new("sup-test://test_mailing_list_header")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('mailing-list-header.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)
@@ -227,10 +221,8 @@ class TestMessage < Minitest::Test
   end
 
   def test_blank_header_lines
-    message = fixture('blank-header-fields.eml')
-
     source = DummySource.new("sup-test://test_blank_header_lines")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('blank-header-fields.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)
@@ -248,11 +240,113 @@ class TestMessage < Minitest::Test
 
   end
 
-  def test_malicious_attachment_names
-    message = fixture('malicious-attachment-names.eml')
+  def test_rfc2047_header_encoding
+    source = DummySource.new("sup-test://test_rfc2047_header_encoding")
+    source.messages = [ fixture_path("rfc2047-header-encoding.eml") ]
+    source_info = 0
+
+    sup_message = Message.build_from_source(source, source_info)
+    sup_message.load_from_source!
+
+    assert_equal("Hans Martin Djupvik, Ingrid Bø, Ирина Сидорова, " +
+                 "Jesper Berg, Frida Engø " +
+                 "bad: =?UTF16?q?badcharsetname?==?US-ASCII?b?/w?=" +
+                 "=?UTF-7?Q?=41=6D=65=72=69=63=61=E2=80=99=73?=",
+                 sup_message.subj)
+  end
+
+  def test_nonascii_header
+    ## Spammers sometimes send invalid high bytes in the headers.
+    ## They will be replaced with U+FFFD REPLACEMENT CHARACTER.
+    source = DummySource.new("sup-test://test_nonascii_header")
+    source.messages = [ fixture_path("non-ascii-header.eml") ]
+    source_info = 0
+
+    sup_message = Message.build_from_source(source, source_info)
+    sup_message.load_from_source!
+
+    assert_equal("SPAM \ufffd", sup_message.from.name)
+    assert_equal("spammer@example.com", sup_message.from.email)
+    assert_equal("spam \ufffd spam", sup_message.subj)
+  end
+
+  def test_utf8_header
+    ## UTF-8 is allowed in header values according to RFC6532.
+    source = DummySource.new("sup-test://test_utf8_header")
+    source.messages = [ fixture_path("utf8-header.eml") ]
+    source_info = 0
+
+    sup_message = Message.build_from_source(source, source_info)
+    sup_message.load_from_source!
+
+    assert_equal(Encoding::UTF_8, sup_message.subj.encoding)
+    assert_equal("LibraryThing: State of the Thing — January", sup_message.subj)
+  end
+
+  def test_nonascii_header_in_nested_message
+    source = DummySource.new("sup-test://test_nonascii_header_in_nested_message")
+    source.messages = [ fixture_path("non-ascii-header-in-nested-message.eml") ]
+    source_info = 0
+
+    sup_message = Message.build_from_source(source, source_info)
+    chunks = sup_message.load_from_source!
+
+    assert_equal(3, chunks.length)
 
+    assert(chunks[0].is_a? Redwood::Chunk::Text)
+
+    assert(chunks[1].is_a? Redwood::Chunk::EnclosedMessage)
+    assert_equal(4, chunks[1].lines.length)
+    assert_equal("From: SPAM \ufffd <spammer@example.com>", chunks[1].lines[0])
+    assert_equal("To: enclosed <enclosed@example.invalid>", chunks[1].lines[1])
+    assert_equal("Subject: spam \ufffd spam", chunks[1].lines[3])
+
+    assert(chunks[2].is_a? Redwood::Chunk::Text)
+    assert_equal(1, chunks[2].lines.length)
+    assert_equal("This is a spam.", chunks[2].lines[0])
+  end
+
+  def test_embedded_message
+    source = DummySource.new("sup-test://test_embedded_message")
+    source.messages = [ fixture_path("embedded-message.eml") ]
+    source_info = 0
+
+    sup_message = Message.build_from_source(source, source_info)
+
+    chunks = sup_message.load_from_source!
+    assert_equal(3, chunks.length)
+
+    assert_equal("sender@example.com", sup_message.from.email)
+    assert_equal("Sender", sup_message.from.name)
+    assert_equal(1, sup_message.to.length)
+    assert_equal("recipient@example.invalid", sup_message.to[0].email)
+    assert_equal("recipient", sup_message.to[0].name)
+    assert_equal("Email with embedded message", sup_message.subj)
+
+    assert(chunks[0].is_a? Redwood::Chunk::Text)
+    assert_equal("Example outer message.", chunks[0].lines[0])
+    assert_equal("Example second line.", chunks[0].lines[1])
+
+    assert(chunks[1].is_a? Redwood::Chunk::EnclosedMessage)
+    assert_equal(4, chunks[1].lines.length)
+    assert_equal("From: Embed sender <embed@example.com>", chunks[1].lines[0])
+    assert_equal("To: rcpt2 <rcpt2@example.invalid>", chunks[1].lines[1])
+    assert_equal("Date: ", chunks[1].lines[2][0..5])
+    assert_equal(
+      Time.rfc2822("Wed, 15 Jul 2020 12:34:56 +0000"),
+      Time.rfc2822(chunks[1].lines[2][6..-1])
+    )
+    assert_equal("Subject: Embedded subject line", chunks[1].lines[3])
+
+    assert(chunks[2].is_a? Redwood::Chunk::Text)
+    assert_equal(2, chunks[2].lines.length)
+    assert_equal("Example embedded message.", chunks[2].lines[0])
+    assert_equal("Second line.", chunks[2].lines[1])
+  end
+
+  def test_malicious_attachment_names
     source = DummySource.new("sup-test://test_blank_header_lines")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('malicious-attachment-names.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)
@@ -276,10 +370,8 @@ class TestMessage < Minitest::Test
     # tries to do the right thing and reply after the quote.
     # In this case we want to just look at the > markers when determining where
     # the quoted chunk ends.
-    message = fixture('zimbra-quote-with-bottom-post.eml')
-
     source = DummySource.new("sup-test://test_zimbra_quote_with_bottom_post")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('zimbra-quote-with-bottom-post.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)