rubymta 0.0.9 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4449c3d949eae74a0d181c0ed70dff9bc2077328
-  data.tar.gz: a2c5a0c3875372065733b1ef4bc9c8ec841f9350
+  metadata.gz: 1601cb4b77cb390064b8a90aaec523a6bdf3c993
+  data.tar.gz: 764b1ccf31bfbb1b3386f9ec42a61510ee5a7edc
 SHA512:
-  metadata.gz: 6efff36106006c5639cbb69aa3688bfeff46911b4550827e2ec778c33793d6a900f38a48b8be491b71b5f0af9186a566a7ff2b05362136ad470112cbf6c23561
-  data.tar.gz: d3220cd027a8c4f40102374ebfcce0c6657f1d741f9e2b076e1c77a6f266dd4a4d606b8c179ea9c728da0e3e97038548fd4f800aa0f7bb35972242ef2f39f08b
+  metadata.gz: eda93abc4e139bec74e5ef868ff3ed2449832557c24948856158a51ad170ec0e0dde2e88feb81dadad1b6e71d160646ed09501d118bebbe14e018556255cb88f
+  data.tar.gz: f1c860ab3f0e7b44da1bc34109e6a2583a641521a30a64eed331894168ca05f42a831e6c0d652d6f0d16a445a88401d1b1eb27bf6a137b89a1605c0dabdfbb73

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+# v0.0.10
+
+#### Warning: This version changes the queue file format. Previous queue files will *not* be compatible with this version.
+
+* Added a test for valid UTF-8 encoding. On a 777 character Greek text, a `valid_encoding?` takes only in 3.6ns, so checking every line of text is not expensive.
+* Revised the way emails are stored on disk: the main structure has the mail[:data][:text] removed and stored following the main structure. The reason for this change is that `eval` doesn't handle foreign UTF-8 character sequences well.
+	- *__New Disk Format in `queue`:__*
+	- Number of lines in main mail structure;
+	- Main mail structure;
+	- Unmodified text; this is added back to the main structure at `mail[:data][:text]` after the queue file is read.
+* Added a Quit at the moment the connecting IP is placed under prohibition. The reason for this is that some senders don't stop sending when they receive a 500+ level message.
+* If there is a `parcel` record, but no matching disk file, the parcel record is marked as delivery='none'. This prevents the QueueRunner from looping when an undelivered queue file is manually deleted.
+* Added checks at the end of delivery in QueueRunner to test for the message level (should be 200+ if accepted).
+* Added another check for the client abruptly closing the connection.
+* Added '=' to the list of acceptable characters in a local part of an email address.
+
 # v0.0.9
 * Added a `rescue OpenSSL::SSL::SSLError` to catch sender certificate violations.
 * Added a `LogLevel` into the Config file (config.rb) to control the logger output. I use LOGGER::INFO as my default.

data/lib/rubymta/contact.rb

@@ -60,6 +60,8 @@ class Contact < Hash
       if prohibited?
         self[:locks] += 1
         self[:expires_at] = Time.now + ProhibitedSeconds
+        modify
+        raise Quit # force quit: some senders try to keep going
       end
       modify
     end

data/lib/rubymta/item_of_mail.rb

@@ -87,13 +87,19 @@ class ItemOfMail < Hash
   end
 
   def save_mail_into_queue_folder
+    # split the ItemOfMail into a hash and a text (which may contain any UTF-8)
+    hash = self.dup
+    text = hash[:data].delete(:text)
     begin
-      # save the mail in the Queue folder
+      # save the mail into the Queue folder
       File::open("#{MailQueue}/#{self[:mail_id]}","w") do |f|
-        self[:saved] = true
-        f.write(self.pretty_inspect)
+        tmp = hash.pretty_inspect
+        f.write("#{tmp.lines.count}\n")
+        f.write(tmp)
+        f.write("\n")
+        f.write(text.join("\n"))
       end
-      return true
+      self[:saved] = true
     rescue => e
       LOG.error(self[:mail_id]) {e.to_s}
       e.backtrace.each { |line| LOG.error(self[:mail_id]) {line} }
@@ -104,14 +110,33 @@ class ItemOfMail < Hash
   def self::retrieve_mail_from_queue_folder(mail_id)
     item = nil
     begin
-      mail = nil
-      File::open("#{MailQueue}/#{mail_id}","r") do |f|
-        mail = eval(f.read)
-        item = ItemOfMail::new(mail)
+      # Make sure the file that matches the parcel record exists
+      if !File::exist?("#{MailQueue}/#{mail_id}")
+        # file missing: mark the parcels none and date them
+        parcels = S3DB[:parcels].where(:mail_id=>mail_id).all
+        parcels.each do |parcel|
+          S3DB[:parcels].where(:id=>parcel[:id]).update(:delivery=>'none', :delivery_at=>Time.now)
+        end
+        return nil
       end
+
+      tmp = nil; File::open("#{MailQueue}/#{mail_id}","r") { |f| tmp = f.read }
+      data = tmp.split("\n")
+
+      # get the number of lines in the hash, and cut that out first
+      n = data[0].to_i
+      a = data[1..n]
+      b = data[n+1..-1]
+
+      # convert the hash
+      mail = eval(a.join("\n"))
+
+      # create the ItemOfMail structure and insert the text
+      item = ItemOfMail::new(mail)
+      item[:data][:text] = b
     rescue => e
-      LOG.error(self[:mail_id]) {e.to_s}
-      e.backtrace.each { |line| LOG.error(self[:mail_id]) {line} }
+      LOG.error(mail_id) {e.to_s}
+      e.backtrace.each { |line| LOG.error(mail_id) {line} }
     end
     return item
   end

data/lib/rubymta/queue_runner.rb

@@ -65,6 +65,7 @@ class QueueRunner
       # sqlite3 has a bug: "<=" doesn't work with time, ex. "retry_at<='#{Time.now}'"
       # we have to add 1 second and use "<"; ex. "retry_at<'#{Time.now+1}'"
       parcels = S3DB[:parcels].where(Sequel.lit("(delivery<>'none') and (delivery_at is null) and ((retry_at is null) or (retry_at<'#{Time.now + 1}'))")).all
+
       return if parcels.empty?
 
       # aggregate the emails by destination domain
@@ -81,21 +82,17 @@ class QueueRunner
       # handled in the respective mail routine
       mail = {}
       deliver.each do |mail_id, domains|
-        (mail = ItemOfMail::retrieve_mail_from_queue_folder(mail_id)) if mail[:mail_id]!=mail_id
-        domains.each do |domain, parcels|
+        (mail = ItemOfMail::retrieve_mail_from_queue_folder(mail_id)) if mail && mail[:mail_id]!=mail_id
 
-#=== compare before and after ======================================
-#puts "--> *2* domain=>#{domain.inspect}"
-#parcels.values.each { |parcel| puts "--> *3* #{parcel.inspect}" }
-#===================================================================
+        if mail.nil?
+          LOG.info(Time.now.strftime("%Y-%m-%d %H:%M:%S")) {"Skipping mail #{mail_id} because there's no associated queue file"}
+          next
+        end
 
+        domains.each do |domain, parcels|
           @mail_id = mail[:mail_id]
           deliver_and_save_status(mail, domain, parcels.values)
           @mail_id = nil
-
-#===================================================================
-#parcels.values.each { |parcel| puts "--> *4* #{parcel.inspect}" }
-#===================================================================
         end
       end
       if (n-=1)<0
@@ -289,7 +286,13 @@ class QueueRunner
 
     # get one final message for all parcels (recipients)
     ok, lines = recv_text
-    return mark_parcels(parcels, lines)
+    if ok=='2'
+      ret = mark_parcels(parcels, lines)
+      LOG.info(@mail_id) {"Mail for #{parcels[0][:to_url]}, et.al. delivered remotely"}
+    else
+      LOG.info(@mail_id) {"Mail for #{parcels[0][:to_url]}, et.al. failed delivery remotely, #{lines.last}"}
+    end
+    return ret
   end
 
 # SAMPLE LMTP TRANSFER
@@ -371,7 +374,13 @@ class QueueRunner
     parcels.each do |parcel|
       # get the response from DoveCot
       ok, lines = recv_text
-      mark_parcels([parcel], lines)
+      if ok=='2'
+        ret = mark_parcels([parcel], lines)
+        LOG.info(@mail_id) {"Mail for #{parcel[:to_url]} delivered locally"}
+      else
+        LOG.info(@mail_id) {"Mail for #{parcel[:to_url]} failed delivery locally, #{lines.last}"}
+      end
+      return ret
     end
   end
 

data/lib/rubymta/receiver.rb

@@ -76,9 +76,14 @@ class Receiver
       Timeout.timeout(ReceiverTimeout) do
         begin
           temp = @connection.gets
-          if temp.nil?
+          case
+          when temp.nil?
             LOG.warn(@mail[:mail_id]) {"The client abruptly closed the connection"}
             text = nil
+          when !temp.valid_encoding?
+            LOG.warn(@mail[:mail_id]) {"The client sent non-UTF-8 text"}
+            send_text("500 5.5.1 non-UTF-8 text detected")
+            raise Quit
           else
             text = temp.chomp
           end
@@ -138,7 +143,7 @@ class Receiver
 #    Character . provided that it is not the first or last character,
 #     and provided also that it does not appear two or more times consecutively.
     part[:dot_error] = true if (local_part[0]=='.' || local_part[-1]=='.' || local_part.index('..'))
-    m = local_part.match(/^[a-zA-Z0-9\!\#\$%&'*+-\/?^_`{|}~]+$/)
+    m = local_part.match(/^[a-zA-Z0-9\!\#\$%&'*+-\/?^_`{|}~=]+$/)
     part[:char_error] = m.nil?
 
     # lookup the email to see if it's one of ours
@@ -352,7 +357,7 @@ class Receiver
            "550 5.1.7 beginning or ending '.' or 2 or more '.'s in a row" \
       if from[:dot_error]
     return "550-5.1.7 #{from[:local_part].inspect} can only", \
-           "550 5.1.7 contain a-z, A_Z, 0-9, and !#\$%&'*+-/?^_`{|}~." \
+           "550 5.1.7 contain a-z, A_Z, 0-9, and !#\$%&'*+-/?^_`{|}~.=" \
       if from[:char_error]
 
     LOG.info(@mail[:mail_id]) {"Receiving mail from sender #{from[:url]}"}

data/lib/rubymta/version.rb

@@ -1,5 +1,5 @@
 module Version
-  VERSION = "0.0.9"
-  MODIFIED = "2017-10-14"
+  VERSION = "0.0.10"
+  MODIFIED = "2017-10-18"
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rubymta
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.10
 platform: ruby
 authors:
 - Michael J. Welch, Ph.D.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-14 00:00:00.000000000 Z
+date: 2017-10-18 00:00:00.000000000 Z
 dependencies: []
 description: RubyMta is an experimental mail transport agent written in Ruby. See
   the README.