sup 0.22.1 → 1.1

data/test/test_message.rb

@@ -21,10 +21,8 @@ class TestMessage < Minitest::Test
   end
 
   def test_simple_message
-    message = fixture('simple-message.eml')
-
     source = DummySource.new("sup-test://test_simple_message")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('simple-message.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)
@@ -99,10 +97,8 @@ class TestMessage < Minitest::Test
   end
 
   def test_multipart_message
-    message = fixture('multi-part.eml')
-
     source = DummySource.new("sup-test://test_multipart_message")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('multi-part.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)
@@ -126,10 +122,8 @@ class TestMessage < Minitest::Test
   end
 
   def test_broken_message_1
-    message = fixture('missing-from-to.eml')
-
     source = DummySource.new("sup-test://test_broken_message_1")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('missing-from-to.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)
@@ -150,10 +144,8 @@ class TestMessage < Minitest::Test
   end
 
   def test_broken_message_2
-    message = fixture('no-body.eml')
-
     source = DummySource.new("sup-test://test_broken_message_1")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('no-body.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)
@@ -167,10 +159,8 @@ class TestMessage < Minitest::Test
   end
 
   def test_multipart_message_2
-    message = fixture('multi-part-2.eml')
-
     source = DummySource.new("sup-test://test_multipart_message_2")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('multi-part-2.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)
@@ -178,14 +168,61 @@ class TestMessage < Minitest::Test
 
     chunks = sup_message.load_from_source! # read the message body chunks
 
-    # TODO: Add more asserts
+    assert_equal(1, chunks.length)
+    assert(chunks[0].is_a? Redwood::Chunk::Attachment)
   end
 
-  def test_blank_header_lines
-    message = fixture('blank-header-fields.eml')
+  def test_text_attachment_decoding
+    source = DummySource.new("sup-test://test_text_attachment_decoding")
+    source.messages = [ fixture_path('text-attachments-with-charset.eml') ]
+    source_info = 0
+
+    sup_message = Message.build_from_source(source, source_info)
+    sup_message.load_from_source!
+
+    chunks = sup_message.load_from_source!
+    assert_equal(7, chunks.length)
+    assert(chunks[0].is_a? Redwood::Chunk::Text)
+    ## The first attachment declares charset=us-ascii
+    assert(chunks[1].is_a? Redwood::Chunk::Attachment)
+    assert_equal(["This is ASCII"], chunks[1].lines)
+    ## The second attachment declares charset=koi8-r and has some Cyrillic
+    assert(chunks[2].is_a? Redwood::Chunk::Attachment)
+    assert_equal(["\u041f\u0440\u0438\u0432\u0435\u0442"], chunks[2].lines)
+    ## The third attachment declares charset=utf-8 and has an emoji
+    assert(chunks[3].is_a? Redwood::Chunk::Attachment)
+    assert_equal(["\u{1f602}"], chunks[3].lines)
+    ## The fourth attachment declares no charset and has a non-ASCII byte,
+    ## which will be replaced with U+FFFD REPLACEMENT CHARACTER
+    assert(chunks[4].is_a? Redwood::Chunk::Attachment)
+    assert_equal(["Embedded\ufffdgarbage"], chunks[4].lines)
+    ## The fifth attachment has an invalid charset, which should still
+    ## be handled gracefully
+    assert(chunks[5].is_a? Redwood::Chunk::Attachment)
+    assert_equal(["Example invalid charset"], chunks[5].lines)
+    ## The sixth attachment is UTF-7 encoded
+    assert(chunks[6].is_a? Redwood::Chunk::Attachment)
+    assert_equal(["This is ✨UTF-7✨"], chunks[6].lines)
+  end
+
+  def test_mailing_list_header
+    source = DummySource.new("sup-test://test_mailing_list_header")
+    source.messages = [ fixture_path('mailing-list-header.eml') ]
+    source_info = 0
 
+    sup_message = Message.build_from_source(source, source_info)
+    sup_message.load_from_source!
+
+    assert(sup_message.list_subscribe.nil?)
+    assert_equal("<https://lists.openembedded.org/g/openembedded-devel/unsub>",
+                 sup_message.list_unsubscribe)
+    assert_equal("openembedded-devel@lists.openembedded.org", sup_message.list_address.email)
+    assert_equal("openembedded-devel", sup_message.list_address.name)
+  end
+
+  def test_blank_header_lines
     source = DummySource.new("sup-test://test_blank_header_lines")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('blank-header-fields.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)
@@ -203,11 +240,113 @@ class TestMessage < Minitest::Test
 
   end
 
-  def test_malicious_attachment_names
-    message = fixture('malicious-attachment-names.eml')
+  def test_rfc2047_header_encoding
+    source = DummySource.new("sup-test://test_rfc2047_header_encoding")
+    source.messages = [ fixture_path("rfc2047-header-encoding.eml") ]
+    source_info = 0
+
+    sup_message = Message.build_from_source(source, source_info)
+    sup_message.load_from_source!
+
+    assert_equal("Hans Martin Djupvik, Ingrid Bø, Ирина Сидорова, " +
+                 "Jesper Berg, Frida Engø " +
+                 "bad: =?UTF16?q?badcharsetname?==?US-ASCII?b?/w?=" +
+                 "=?UTF-7?Q?=41=6D=65=72=69=63=61=E2=80=99=73?=",
+                 sup_message.subj)
+  end
+
+  def test_nonascii_header
+    ## Spammers sometimes send invalid high bytes in the headers.
+    ## They will be replaced with U+FFFD REPLACEMENT CHARACTER.
+    source = DummySource.new("sup-test://test_nonascii_header")
+    source.messages = [ fixture_path("non-ascii-header.eml") ]
+    source_info = 0
+
+    sup_message = Message.build_from_source(source, source_info)
+    sup_message.load_from_source!
 
+    assert_equal("SPAM \ufffd", sup_message.from.name)
+    assert_equal("spammer@example.com", sup_message.from.email)
+    assert_equal("spam \ufffd spam", sup_message.subj)
+  end
+
+  def test_utf8_header
+    ## UTF-8 is allowed in header values according to RFC6532.
+    source = DummySource.new("sup-test://test_utf8_header")
+    source.messages = [ fixture_path("utf8-header.eml") ]
+    source_info = 0
+
+    sup_message = Message.build_from_source(source, source_info)
+    sup_message.load_from_source!
+
+    assert_equal(Encoding::UTF_8, sup_message.subj.encoding)
+    assert_equal("LibraryThing: State of the Thing — January", sup_message.subj)
+  end
+
+  def test_nonascii_header_in_nested_message
+    source = DummySource.new("sup-test://test_nonascii_header_in_nested_message")
+    source.messages = [ fixture_path("non-ascii-header-in-nested-message.eml") ]
+    source_info = 0
+
+    sup_message = Message.build_from_source(source, source_info)
+    chunks = sup_message.load_from_source!
+
+    assert_equal(3, chunks.length)
+
+    assert(chunks[0].is_a? Redwood::Chunk::Text)
+
+    assert(chunks[1].is_a? Redwood::Chunk::EnclosedMessage)
+    assert_equal(4, chunks[1].lines.length)
+    assert_equal("From: SPAM \ufffd <spammer@example.com>", chunks[1].lines[0])
+    assert_equal("To: enclosed <enclosed@example.invalid>", chunks[1].lines[1])
+    assert_equal("Subject: spam \ufffd spam", chunks[1].lines[3])
+
+    assert(chunks[2].is_a? Redwood::Chunk::Text)
+    assert_equal(1, chunks[2].lines.length)
+    assert_equal("This is a spam.", chunks[2].lines[0])
+  end
+
+  def test_embedded_message
+    source = DummySource.new("sup-test://test_embedded_message")
+    source.messages = [ fixture_path("embedded-message.eml") ]
+    source_info = 0
+
+    sup_message = Message.build_from_source(source, source_info)
+
+    chunks = sup_message.load_from_source!
+    assert_equal(3, chunks.length)
+
+    assert_equal("sender@example.com", sup_message.from.email)
+    assert_equal("Sender", sup_message.from.name)
+    assert_equal(1, sup_message.to.length)
+    assert_equal("recipient@example.invalid", sup_message.to[0].email)
+    assert_equal("recipient", sup_message.to[0].name)
+    assert_equal("Email with embedded message", sup_message.subj)
+
+    assert(chunks[0].is_a? Redwood::Chunk::Text)
+    assert_equal("Example outer message.", chunks[0].lines[0])
+    assert_equal("Example second line.", chunks[0].lines[1])
+
+    assert(chunks[1].is_a? Redwood::Chunk::EnclosedMessage)
+    assert_equal(4, chunks[1].lines.length)
+    assert_equal("From: Embed sender <embed@example.com>", chunks[1].lines[0])
+    assert_equal("To: rcpt2 <rcpt2@example.invalid>", chunks[1].lines[1])
+    assert_equal("Date: ", chunks[1].lines[2][0..5])
+    assert_equal(
+      Time.rfc2822("Wed, 15 Jul 2020 12:34:56 +0000"),
+      Time.rfc2822(chunks[1].lines[2][6..-1])
+    )
+    assert_equal("Subject: Embedded subject line", chunks[1].lines[3])
+
+    assert(chunks[2].is_a? Redwood::Chunk::Text)
+    assert_equal(2, chunks[2].lines.length)
+    assert_equal("Example embedded message.", chunks[2].lines[0])
+    assert_equal("Second line.", chunks[2].lines[1])
+  end
+
+  def test_malicious_attachment_names
     source = DummySource.new("sup-test://test_blank_header_lines")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('malicious-attachment-names.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)
@@ -225,6 +364,33 @@ class TestMessage < Minitest::Test
   # TODO: test different quoting styles, see that they are all divided
   # to chunks properly
 
+  def test_zimbra_quote_with_bottom_post
+    # Zimbra does an Outlook-style "Original Message" delimiter and then *also*
+    # prefixes each quoted line with a > marker. That's okay until the sender
+    # tries to do the right thing and reply after the quote.
+    # In this case we want to just look at the > markers when determining where
+    # the quoted chunk ends.
+    source = DummySource.new("sup-test://test_zimbra_quote_with_bottom_post")
+    source.messages = [ fixture_path('zimbra-quote-with-bottom-post.eml') ]
+    source_info = 0
+
+    sup_message = Message.build_from_source(source, source_info)
+    chunks = sup_message.load_from_source!
+
+    assert_equal(3, chunks.length)
+
+    # TODO this chunk should ideally be part of the quote chunk after it.
+    assert(chunks[0].is_a? Redwood::Chunk::Text)
+    assert_equal(1, chunks[0].lines.length)
+    assert_equal("----- Original Message -----", chunks[0].lines.first)
+
+    assert(chunks[1].is_a? Redwood::Chunk::Quote)
+
+    assert(chunks[2].is_a? Redwood::Chunk::Text)
+    assert_equal(3, chunks[2].lines.length)
+    assert_equal("This is the reply from the Zimbra user.",
+                 chunks[2].lines[2])
+  end
 end
 
 end

data/test/test_messages_dir.rb

@@ -13,19 +13,22 @@ class TestMessagesDir < ::Minitest::Test
   def setup
     @path = Dir.mktmpdir
     Redwood::HookManager.init File.join(@path, 'hooks')
+    @log = StringIO.new
+    Redwood::Logger.add_sink @log
+    Redwood::Logger.remove_sink $stderr
   end
 
   def teardown
+    Redwood::Logger.clear!
+    Redwood::Logger.remove_sink @log
+    Redwood::Logger.add_sink $stderr
     Redwood::HookManager.deinstantiate!
     FileUtils.rm_r @path
   end
 
   def test_binary_content_transfer_encoding
-    message = ''
-    File.open('test/fixtures/binary-content-transfer-encoding-2.eml') { |f| message = f.read }
-
     source = DummySource.new("sup-test://test_messages")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('binary-content-transfer-encoding-2.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)
@@ -41,7 +44,6 @@ class TestMessagesDir < ::Minitest::Test
     assert_equal("Important", subj)
 
     chunks = sup_message.load_from_source!
-    indexable_chunks = sup_message.indexable_chunks
 
     # there should be only one chunk
     #assert_equal(1, chunks.length)
@@ -50,14 +52,13 @@ class TestMessagesDir < ::Minitest::Test
 
     # lines should contain an error message
     assert (lines.join.include? "An error occurred while loading this message."), "This message should not load successfully"
+
+    assert_match(/WARNING: problem reading message/, @log.string)
   end
 
   def test_bad_content_transfer_encoding
-    message = ''
-    File.open('test/fixtures/bad-content-transfer-encoding-1.eml') { |f| message = f.read }
-
     source = DummySource.new("sup-test://test_messages")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('bad-content-transfer-encoding-1.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)
@@ -73,7 +74,6 @@ class TestMessagesDir < ::Minitest::Test
     assert_equal("Content-Transfer-Encoding:-bug in sup", subj)
 
     chunks = sup_message.load_from_source!
-    indexable_chunks = sup_message.indexable_chunks
 
     # there should be only one chunk
     #assert_equal(1, chunks.length)
@@ -82,14 +82,13 @@ class TestMessagesDir < ::Minitest::Test
 
     # lines should contain an error message
     assert (lines.join.include? "An error occurred while loading this message."), "This message should not load successfully"
+
+    assert_match(/WARNING: problem reading message/, @log.string)
   end
 
   def test_missing_line
-    message = ''
-    File.open('test/fixtures/missing-line.eml') { |f| message = f.read }
-
     source = DummySource.new("sup-test://test_messages")
-    source.messages = [ message ]
+    source.messages = [ fixture_path('missing-line.eml') ]
     source_info = 0
 
     sup_message = Message.build_from_source(source, source_info)
@@ -105,7 +104,6 @@ class TestMessagesDir < ::Minitest::Test
     assert_equal("Encoding bug", subj)
 
     chunks = sup_message.load_from_source!
-    indexable_chunks = sup_message.indexable_chunks
 
     # there should be only one chunk
     #assert_equal(1, chunks.length)

data/test/unit/test_horizontal_selector.rb

@@ -13,18 +13,18 @@ describe Redwood::HorizontalSelector do
 
   it "init w/ the first value selected" do
     first_value = values.first
-    @selector.val.must_equal first_value
+    assert_equal first_value, @selector.val
   end
 
   it "stores value for selection" do
     second_value = values[1]
     @selector.set_to second_value
-    @selector.val.must_equal second_value
+    assert_equal second_value, @selector.val
   end
 
   describe "for unknown value" do
     it "cannot select unknown value" do
-      @selector.wont_be :can_set_to?, strange_value
+      assert_equal false, @selector.can_set_to?(strange_value)
     end
 
     it "refuses selecting unknown value" do
@@ -34,7 +34,7 @@ describe Redwood::HorizontalSelector do
         @selector.set_to strange_value
       end
 
-      @selector.val.must_equal old_value
+      assert_equal old_value, @selector.val
     end
   end
 end

data/test/unit/test_locale_fiddler.rb

@@ -1,7 +1,7 @@
 require 'test_helper'
 require 'sup/util/locale_fiddler'
 
-class TestFiddle < ::Minitest::Unit::TestCase
+class TestFiddle < Minitest::Test
   # TODO this is a silly test
   def test_fiddle_set_locale
     before = LocaleDummy.setlocale(6, nil).to_s

data/test/unit/util/test_query.rb

@@ -24,9 +24,14 @@ describe Redwood::Util::Query do
       query = Xapian::Query.new msg
       life = 'hæi'
 
-      # this is now possibly UTF-8 string with possibly invalid chars
-      assert_raises Redwood::Util::Query::QueryDescriptionError do
-        desc = Redwood::Util::Query.describe (query)
+      if query.description.force_encoding("UTF-8").valid_encoding?
+        # xapian 1.4 internally handles this bad input
+        assert true
+      else
+        # xapian 1.2 doesn't handle this bad input, so we do
+        assert_raises Redwood::Util::Query::QueryDescriptionError do
+          _desc = Redwood::Util::Query.describe (query)
+        end
       end
 
       assert_raises Encoding::CompatibilityError do
@@ -40,7 +45,8 @@ describe Redwood::Util::Query do
 
       desc = Redwood::Util::Query.describe(query, "invalid query")
 
-      assert_equal("invalid query", desc)
+      assert desc.force_encoding("UTF-8").valid_encoding?
+
     end
   end
 end

data/test/unit/util/test_string.rb

@@ -11,12 +11,14 @@ describe "Sup's String extension" do
         ['some words', 10,],
         ['中文', 4,],
         ['ä', 1,],
+        ['😱', 2],
+        #['🏳️‍🌈', 2],  # Emoji ZWJ sequence not yet supported (see PR #563)
       ]
     end
 
     it "calculates display length of a string" do
       data.each do |(str, length)|
-        str.display_length.must_equal length
+        assert_equal length, str.display_length
       end
     end
   end
@@ -27,12 +29,14 @@ describe "Sup's String extension" do
         ['some words', 6, 'some w'],
         ['中文', 2, '中'],
         ['älpha', 3, 'älp'],
+        ['😱😱', 2, '😱'],
+        #['🏳️‍🌈', 2, '🏳️‍🌈'],  # Emoji ZWJ sequence not yet supported (see PR #563)
       ]
     end
 
     it "slices string by display length" do
       data.each do |(str, length, sliced)|
-        str.slice_by_display_length(length).must_equal sliced
+        assert_equal sliced, str.slice_by_display_length(length)
       end
     end
   end
@@ -45,12 +49,14 @@ describe "Sup's String extension" do
         ['中文', 2, ['中', '文']],
         ['中文', 5, ['中文']],
         ['älpha', 3, ['älp', 'ha']],
+        ['😱😱', 2, ['😱', '😱']],
+        #['🏳️‍🌈🏳️‍🌈', 2, ['🏳️‍🌈', '🏳️‍🌈']],  # Emoji ZWJ sequence not yet supported (see PR #563)
       ]
     end
 
     it "wraps string by display length" do
       data.each do |(str, length, wrapped)|
-        str.wrap(length).must_equal wrapped
+        assert_equal wrapped, str.wrap(length)
       end
     end
   end

data/test/unit/util/test_uri.rb

@@ -7,13 +7,13 @@ describe Redwood::Util::Uri do
     it "builds uri from hash" do
       components = {:path => "/var/mail/foo", :scheme => "mbox"}
       uri = Redwood::Util::Uri.build(components)
-      uri.to_s.must_equal "mbox:/var/mail/foo"
+      assert_equal "mbox:/var/mail/foo", uri.to_s
     end
 
     it "expands ~ in path" do
       components = {:path => "~/foo", :scheme => "maildir"}
       uri = Redwood::Util::Uri.build(components)
-      uri.to_s.must_equal "maildir:#{ENV["HOME"]}/foo"
+      assert_equal "maildir:#{ENV["HOME"]}/foo", uri.to_s
     end
   end
 end