sisimai 4.22.0 → 4.22.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 78125bd98eedb70240490fd0dfef3e3072679204
-  data.tar.gz: c384d1e57686bd86686338c5877ee1197ffb4272
+  metadata.gz: c644b2582fd163b4227e472c750409fff659c691
+  data.tar.gz: 39c38f411e6411a77f907d96877c808a8f731879
 SHA512:
-  metadata.gz: 022a3627ae60a191f1eb641f2d5dac85672806a0453b99cbc19b27d5057a2845054dae8f4b7bbd441199c9617e5660de8b9dffbc4fb898903b3bdecd7af5b397
-  data.tar.gz: 7d294183cfd12dbe5e8dbf1de142ffd94897039d12ec0f5d0bd9ac200a588ea7aa9bda7ac31f6fdd695a87852d9695d5dceb5ccbe3e80aa6741936d1f73b05f5
+  metadata.gz: 8d08ac534acade16e7aedd5ef71bd12bd44702b51283833a874e1f178f4e3e5e7e6bab3f8544e0e1254e686f8c4455e15b397bbd05a89a2aba343202013a818e
+  data.tar.gz: 2302808a843715d2a463256abde74c8dd208b277d4325941b39e64ad87f429252e65f794e7a30fd8a924e2d665b77e7026088cbd974a9fcfc29b61f67c5e45be

data/.travis.yml

@@ -5,8 +5,8 @@ rvm:
   - 2.2.4
   - 2.3.0
   - 2.4.0
-  - jruby-9.0.4.0
   - jruby-9.0.5.0
+  - jruby-9.1.9.0
 before_install:
 script:
   - rake spec

data/ChangeLog.md

@@ -3,6 +3,26 @@ RELEASE NOTES for Ruby version of Sisimai
 - releases: "https://github.com/sisimai/rb-Sisimai/releases"
 - download: "https://rubygems.org/gems/sisimai"
 
+v4.22.1
+--------------------------------------------------------------------------------
+- release: "Tue, 29 Aug 2017 17:25:22 +0900 (JST)"
+- version: "4.22.1"
+- changes:
+  - Sisimai::Address was born again: import Pull-Request sisimai/p5-Sisimai#231
+    - Implement new email address parser method: find()
+    - Implement new constructor: make()
+    - Implement new writable accessors: name() and comment()
+    - parse() method was marked as obsoleted
+  - **Require Oj 3.0.0 or later. Build test fails when the version of installed
+    Oj is 2.18.* on CRuby.**
+  - Tested on JRuby 9.1.9.0.
+  - Fix wrong constant name in Sisimai::Rhost::ExchangeOnline reported at issue
+    #77. Thanks to @rdeavila.
+  - Improved code in Sisimai::Message::Email to avoid an exception reported at
+    issue #82. Thanks to @hiroyuki-sato.
+  - Fixed wrong bitwise operation in Sisimai::RFC3464 for getting the original
+    message part Thanks to @hiroyuki-sato.
+
 v4.22.0
 --------------------------------------------------------------------------------
 - release: "Tue, 22 Aug 2017 18:25:55 +0900 (JST)"

data/README-JA.md

@@ -239,8 +239,8 @@ Differences between Ruby version and Perl version
 | メール解析速度(1000通のメール)              | 3.30秒         | 2.33秒        |
 | インストール方法                            | gem install    | cpanm, cpm    |
 | 依存モジュール数(コアモジュールを除く)      | 1モジュール    | 2モジュール   |
-| LOC:ソースコードの行数                      | 12400行        | 9100行        |
-| テスト件数(spec/,t/,xt/ディレクトリ)        | 205700件       | 202400件      |
+| LOC:ソースコードの行数                      | 12600行        | 9300行        |
+| テスト件数(spec/,t/,xt/ディレクトリ)        | 196800件       | 204600件      |
 | ライセンス                                  | 二条項BSD      | 二条項BSD     |
 | 開発会社によるサポート契約                  | 準備中         | 提供中        |
 

data/README.md

@@ -243,8 +243,8 @@ and bounceHammer are available at
 | The speed of parsing email(1000 emails)     | 3.30s          | 2.33s         |
 | How to install                              | gem install    | cpanm, cpm    |
 | Dependencies (Except core modules)          | 1 module       | 2 modules     |
-| LOC:Source lines of code                    | 12400 lines    | 9100 lines    |
-| The number of tests(spec/,t/,xt/) directory | 205700 tests   | 202400 tests  |
+| LOC:Source lines of code                    | 12600 lines    | 9300 lines    |
+| The number of tests(spec/,t/,xt/) directory | 196800 tests   | 204600 tests  |
 | License                                     | BSD 2-Clause   | BSD 2-Clause  |
 | Support Contract provided by Developer      | Coming soon    | Available     |
 

data/lib/sisimai/address.rb

@@ -16,75 +16,102 @@ module Sisimai
       return sprintf('undisclosed-%s-in-headers@libsisimai.org.invalid', local)
     end
 
-    def self.find(argvs, addrs = nil)
+    # New constructor of Sisimai::Address
+    # @param    [Hash] argvs        Email address, name, and other elements
+    # @return   [Sisimai::Address]  Object or Undef when the email address was
+    #                               not valid.
+    # @example  make({address: 'neko@example.org', name: 'Neko', comment: '(nyaan)')}
+    #           # => Sisimai::Address object
+    def self.make(argvs)
+      return nil unless argvs.is_a? Hash
+      return nil unless argvs.key?(:address)
+      return nil if argvs[:address].empty?
+
+      thing = Sisimai::Address.new(argvs[:address])
+      return nil unless thing
+      return nil if thing.void
+
+      thing.name    = argvs[:name]    || ''
+      thing.comment = argvs[:comment] || ''
+
+      return thing
+    end
+
+    def self.find(argv1 = nil, addrs = false)
       # Email address parser with a name and a comment
-      # @param    [String] argvs  String including email address
+      # @param    [String] argv1  String including email address
       # @param    [Boolean] addrs true:  Returns list including all the elements
       #                           false: Returns list including email addresses only
       # @return   [Array, Nil]    Email address list or Undef when there is no 
       #                           email address in the argument
       # @example  Parse email address
-      #   input:  parse('Neko <neko@example.cat>')
-      #   output: [{'address' => 'neko@example.cat', 'name' => 'Neko'}]
-      return nil unless argvs
-      argvs = argvs.gsub(/[\r\n]/, '')
+      #   find('Neko <neko(nyaan)@example.org>')
+      #   #=> [{ address: 'neko@example.org', name: 'Neko', comment: '(nyaan)'}]
+      return nil unless argv1
+      argv1 = argv1.gsub(/[\r\n]/, '')
 
-      emailtable = { 'address' => '', 'name' => '', 'comment' => '' }
+      emailtable = { address: '', name: '', comment: '' }
       addrtables = []
+      readbuffer = []
       readcursor = 0
       delimiters = ['<', '>', '(', ')', '"', ',']
+      validemail = %r{(?>
+        (?:([^\s]+|["].+?["]))          # local part
+        [@]
+        (?:([^@\s]+|[0-9A-Za-z:\.]+))   # domain part
+        )
+      }x
       indicators = {
         :'email-address' => (1 << 0),    # <neko@example.org>
         :'quoted-string' => (1 << 1),    # "Neko, Nyaan"
         :'comment-block' => (1 << 2),    # (neko)
       }
-      characters = argvs.split('')
-      readbuffer = []
 
       v = emailtable  # temporary buffer
       p = ''          # current position
 
-      characters.each do |e|
+      argv1.split('').each do |e|
         # Check each characters
+
         if delimiters.detect { |r| r == e }
           # The character is a delimiter character
           if e == ','
             # Separator of email addresses or not
-            if v['address'] =~ /\A[<].+[@].+[>]\z/
+            if v[:address] =~ /\A[<].+[@].+[>]\z/
               # An email address has already been picked
 
               if readcursor & indicators[:'comment-block'] > 0
                 # The cursor is in the comment block (Neko, Nyaan)
-                v['comment'] += e
+                v[:comment] += e
 
               elsif readcursor & indicators[:'quoted-string'] > 0
                 # "Neko, Nyaan"
-                v['name'] += e
+                v[:name] += e
 
               else
                 # The cursor is not in neither the quoted-string nor the comment block
                 readcursor = 0  # reset cursor position
                 readbuffer << v
-                v = { 'address' => '', 'name' => '', 'comment' => '' }
+                v = { address: '', name: '', comment: '' }
                 p = ''
               end
             else
               # "Neko, Nyaan" <neko@nyaan.example.org> OR <"neko,nyaan"@example.org>
-              p.size > 0 ? (v[p] += e) : (v['name'] += e)
+              p.size > 0 ? (v[p] += e) : (v[:name] += e)
             end
             next
           end # End of if(',')
 
           if e == '<'
             # <: The beginning of an email address or not
-            if v['address'].size > 0
-              p.size > 0 ? (v[p] += e) : (v['name'] += e)
+            if v[:address].size > 0
+              p.size > 0 ? (v[p] += e) : (v[:name] += e)
 
             else
               # <neko@nyaan.example.org>
               readcursor |= indicators[:'email-address']
-              v['address'] += e
-              p = 'address'
+              v[:address] += e
+              p = :address
             end
             next
           end
@@ -95,11 +122,11 @@ module Sisimai
             if readcursor & indicators[:'email-address'] > 0
               # <neko@example.org>
               readcursor &= ~indicators[:'email-address']
-              v['address'] += e
+              v[:address] += e
               p = ''
             else
               # a comment block or a display name
-              p.size > 0 ? (v['comment'] += e) : (v['name'] += e)
+              p.size > 0 ? (v[:comment] += e) : (v[:name] += e)
             end
             next
           end # End of if('>')
@@ -108,29 +135,32 @@ module Sisimai
             # The beginning of a comment block or not
             if readcursor & indicators[:'email-address'] > 0
               # <"neko(nyaan)"@example.org> or <neko(nyaan)@example.org>
-              if v['address'] =~ /["]/
+              if v[:address] =~ /["]/
                 # Quoted local part: <"neko(nyaan)"@example.org>
-                v['address'] += e
+                v[:address] += e
 
               else
                 # Comment: <neko(nyaan)@example.org>
                 readcursor |= indicators[:'comment-block']
-                v['comment'] += e
-                p = 'comment'
+                v[:comment] += ' ' if v[:comment] =~ /[)]\z/
+                v[:comment] += e
+                p = :comment
               end
             elsif readcursor & indicators[:'comment-block'] > 0
               # Comment at the outside of an email address (...(...)
-              v['comment'] += e
+              v[:comment] += ' ' if v[:comment] =~ /[)]\z/
+              v[:comment] += e
 
             elsif readcursor & indicators[:'quoted-string'] > 0
               # "Neko, Nyaan(cat)", Deal as a display name
-              v['name'] += e
+              v[:name] += e
 
             else
               # The beginning of a comment block
               readcursor |= indicators[:'comment-block']
-              v['comment'] += e
-              p = 'comment'
+              v[:comment] += ' ' if v[:comment] =~ /[)]\z/
+              v[:comment] += e
+              p = :comment
             end
             next
           end # End of if('(')
@@ -139,26 +169,26 @@ module Sisimai
             # The end of a comment block or not
             if readcursor & indicators[:'email-address'] > 0
               # <"neko(nyaan)"@example.org> OR <neko(nyaan)@example.org>
-              if v['address'] =~ /["]/
+              if v[:address] =~ /["]/
                 # Quoted string in the local part: <"neko(nyaan)"@example.org>
-                v['address'] += e
+                v[:address] += e
 
               else
                 # Comment: <neko(nyaan)@example.org>
                 readcursor &= ~indicators[:'comment-block']
-                v['comment'] += e
-                p = 'address'
+                v[:comment] += e
+                p = :address
               end
             elsif readcursor & indicators[:'comment-block'] > 0
               # Comment at the outside of an email address (...(...)
               readcursor &= ~indicators[:'comment-block']
-              v['comment'] += e
+              v[:comment] += e
               p = ''
 
             else
               # Deal as a display name
               readcursor &= ~indicators[:'comment-block']
-              v['name'] = e
+              v[:name] = e
               p = ''
             end
             next
@@ -171,92 +201,87 @@ module Sisimai
               v[p] += e
             else
               # Display name
-              v['name'] += e
+              v[:name] += e
               if readcursor & indicators[:'quoted-string'] > 0
                 # "Neko, Nyaan"
-                unless v['name'] =~ /\x5c["]\z/
+                unless v[:name] =~ /\x5c["]\z/
                   # "Neko, Nyaan \"...
                   readcursor &= ~indicators[:'quoted-string']
                   p = ''
                 end
-              else
-                if readcursor & indicators[:'email-address'] == 0 &&
-                   readcursor & indicators[:'comment-block'] == 0
-                  # Deal as the beginning of a display name
-                  readcursor |= indicators[:'quoted-string']
-                  p = 'name'
-                end
               end
             end
             next
           end # End of if('"') 
         else
           # The character is not a delimiter
-          p.size > 0 ? (v[p] += e) : (v['name'] += e)
+          p.size > 0 ? (v[p] += e) : (v[:name] += e)
           next
         end
       end
 
-      if v['address'].size > 0
+      if v[:address].size > 0
         # Push the latest values
         readbuffer << v
       else
         # No email address like <neko@example.org> in the argument
-        if v['name'] =~ /[@]/
+        if cv = v[:name].match(validemail)
           # String like an email address will be set to the value of "address"
-          if cv = v['name'].match(/(["].+?["][@][^ ]+)/)
-            # "neko nyaan"@example.org
-            v['address'] = cv[1]
+          v[:address] = sprintf("%s@%s", cv[1], cv[2])
 
-          elsif cv = v['name'].match(/([^\s]+[@][^\s]+)/)
-            # neko-nyaan@example.org
-            v['address'] = cv[1]
-          end
-        elsif Sisimai::RFC5322.is_mailerdaemon(v['name'])
+        elsif Sisimai::RFC5322.is_mailerdaemon(v[:name])
           # Allow if the argument is MAILER-DAEMON
-          v['address'] = v['name']
+          v[:address] = v[:name]
         end
 
-        if v['address'].size > 0
-          # Remove the value of "name" and remove the comment from the address
-          if cv = v['address'].match(/(.*)([(].+[)])(.*)/)
+        if v[:address].size > 0
+          # Remove the comment from the address
+          if cv = v[:address].match(/(.*)([(].+[)])(.*)/)
             # (nyaan)nekochan@example.org, nekochan(nyaan)cat@example.org or
             # nekochan(nyaan)@example.org
-            v['address'] = cv[1] + cv[3]
-            v['comment'] = cv[2]
+            v[:address] = cv[1] + cv[3]
+            v[:comment] = cv[2]
           end
-          v['name'] = ''
           readbuffer << v
         end
       end
 
       readbuffer.each do |e|
-        unless e['address'] =~ /\A.+[@].+\z/
+        # The element must not include any character except from 0x20 to 0x7e.
+        next if e[:address] =~ /[^\x20-\x7e]/
+
+        unless e[:address] =~ /\A.+[@].+\z/
           # Allow if the argument is MAILER-DAEMON
-          next unless Sisimai::RFC5322.is_mailerdaemon(e['address'])
+          next unless Sisimai::RFC5322.is_mailerdaemon(e[:address])
         end
 
-        e['address'] = e['address'].sub(/\A[\[<{('`]/, '')
-        e['address'] = e['address'].sub(/['`>})\]]\z/, '')
+        # Remove angle brackets, other brackets, and quotations: []<>{}'`
+        # except a domain part is an IP address like neko@[192.0.2.222]
+        e[:address] = e[:address].sub(/\A[\[<{('`]/, '')
+        e[:address] = e[:address].sub(/['`>})]\z/, '')
+        e[:address] = e[:address].sub(/\]\z/, '') unless e[:address] =~ /[@]\[[0-9A-Z:\.]+\]\z/i
 
-        unless e['address'] =~ /\A["].+["][@]/
+        unless e[:address] =~ /\A["].+["][@]/
           # Remove double-quotations
-          e['address'] = e['address'].sub(/\A["]/, '')
-          e['address'] = e['address'].sub(/["]\z/, '')
+          e[:address] = e[:address].sub(/\A["]/, '')
+          e[:address] = e[:address].sub(/["]\z/, '')
         end
 
         if addrs
           # Almost compatible with parse() method, returns email address only
-          e.delete('name')
-          e.delete('comment')
+          e.delete(:name)
+          e.delete(:comment)
         else
           # Remove double-quotations, trailing spaces.
-          %w|name comment|.each do |f|
+          [:name, :comment].each do |f|
             e[f] = e[f].sub(/\A\s*/, '')
             e[f] = e[f].sub(/\s*\z/, '')
           end
-          e['name'] = e['name'].sub(/\A["]/, '')
-          e['name'] = e['name'].sub(/["]\z/, '')
+          e[:comment] = '' unless e[:comment] =~ /\A[(].+[)]/
+
+          e[:name] = e[:name].squeeze(' ')     unless e[:name] =~ /\A["].+["]\z/
+          e[:name] = e[:name].sub(/\A["]/, '') unless e[:name] =~ /\A["].+["][@]/
+          e[:name] = e[:name].sub(/["]\z/, '')
         end
         addrtables << e
       end
@@ -269,93 +294,43 @@ module Sisimai
     # @param    [Array] argvs   List of strings including email address
     # @return   [Array, Nil]    Email address list or Undef when there is no
     #                           email address in the argument
-    # @example  Parse email address
-    #   parse( [ 'Neko <neko@example.cat>' ] )  #=> [ 'neko@example.cat' ]
-    def self.parse(argvs)
+    # @until    v4.22.1
+    def self.parse(argvs = nil)
+      return nil unless argvs
+      return nil unless argvs.is_a? Array
+      return nil if argvs.empty?
+
+      warn ' ***warning: Sisimai::Address.parse is marked as obsoleted'
       addrs = []
+
       argvs.each do |e|
         # Parse each element in the array
         #   1. The element must include '@'.
         #   2. The element must not include character except from 0x20 to 0x7e.
         next unless e
-        unless e =~ /[@]/
-          # Allow if the argument is MAILER-DAEMON
-          next unless Sisimai::RFC5322.is_mailerdaemon(e)
-        end
-        next if e =~ /[^\x20-\x7e]/
+        next unless e.is_a? Object::String
+        next if e.empty?
 
-        v = Sisimai::Address.s3s4(e)
-        if v.size > 0
-          # The element includes a valid email address
-          addrs << v
-        end
+        v = Sisimai::Address.find(e, 1) || []
+        next if v.empty?
+        v.each { |f| addrs << f[:address] }
       end
 
-      return nil unless addrs.size > 0
+      return nil if addrs.empty?
       return addrs
     end
 
     # Runs like ruleset 3,4 of sendmail.cf
-    # @param    [String] email  Text including an email address
+    # @param    [String] input  Text including an email address
     # @return   [String]        Email address without comment, brackets
-    # @example  Parse email address
-    #   s3s4( '<neko@example.cat>' ) #=> 'neko@example.cat'
+    # @example  s3s4('<neko@example.cat>') #=> 'neko@example.cat'
     def self.s3s4(input)
-      unless input =~ /[ ]/
-        # no space character between " and < .
-        # no space character between " and < .
-        input = input.sub(/\A(.+)"<(.+)\z/, '\1" <\2')      # "=?ISO-2022-JP?B?....?="<user@example.org>,
-        input = input.sub(/\A(.+)[?]=<(.+)\z/, '\1?= <\2')  # =?ISO-2022-JP?B?....?=<user@example.org>
-
-        # comment-part<localpart@domainpart>
-        input = input.sub(/[<]/, ' <') unless input =~ /\A[<]/
-        input = input.sub(/[>]/, '> ') unless input =~ /[>]\z/
-      end
-
-      canon = ''
-      addrs = []
-      token = input.split(' ')
-
-      token.map! do |e|
-        # Convert character entity; "&lt;" -> ">", "&gt;" -> "<".
-        e = e.gsub(/&lt;/, '<')
-        e = e.gsub(/&gt;/, '>')
-        e = e.gsub(/,\z/, '')
-      end
-
-      if token.size == 1
-        addrs << token[0]
-
-      else
-        token.each do |e|
-          e.chomp
-          unless e =~ /\A[<]?.+[@][-.0-9A-Za-z]+[.]?[A-Za-z]{2,}[>]?\z/
-            # Check whether the element is mailer-daemon or not
-            next unless Sisimai::RFC5322.is_mailerdaemon(e)
-          end
-          addrs << e
-        end
-      end
-
-      if addrs.size > 1
-        # Get the first element which is <...> format string from @addrs array.
-        canon = addrs.detect { |e| e =~ /\A[<].+[>]\z/ } || ''
-        canon = addrs[0] if canon.size < 1
-
-      else
-        canon = addrs.shift
-      end
+      return nil unless input
+      return input unless input.is_a? Object::String
 
-      return '' if !canon || canon == ''
-      canon = canon.delete('<>[]():;')  # Remove brackets, colons
-
-      if canon =~ /\A["].+["][@].+\z/
-        canon = canon.delete(%q|{}'`|)  # "localpart..."@example.org
-      else
-        canon = canon.delete(%q|{}'"`|) # Remove brackets, quotations
-      end
-
-      return canon
+      addrs = Sisimai::Address.find(input, 1) || []
+      return input if addrs.empty?
+      return addrs[0][:address]
     end
 
     # Expand VERP: Get the original recipient address from VERP
@@ -368,9 +343,8 @@ module Sisimai
       verp0 = ''
 
       if cv = local.match(/\A[-_\w]+?[+](\w[-._\w]+\w)[=](\w[-.\w]+\w)\z/)
-        verp0 = cv[1] + '@' + cv[2]
+        verp0 = sprintf("%s@%s", cv[1], cv[2])
         return verp0 if Sisimai::RFC5322.is_emailaddress(verp0)
-
       else
         return ''
       end
@@ -399,74 +373,68 @@ module Sisimai
       :verp,    # [String] VERP
       :alias,   # [String] alias of the email address
     ]
+    @@rwaccessors = [
+      :name,    # [String] Display name
+      :comment, # [String] Comment
+    ]
     @@roaccessors.each { |e| attr_reader   e }
+    @@rwaccessors.each { |e| attr_accessor e }
 
     # Constructor of Sisimai::Address
-    # @param <str>  [String] email          Email address
+    # @param <str>  [String] argv1          Email address
     # @return       [Sisimai::Address, Nil] Object or Undef when the email
     #                                       address was not valid
-    def initialize(email)
-      return nil unless email
+    def initialize(argv1)
+      return nil unless argv1
+
+      addrs = Sisimai::Address.find(argv1)
+      return nil unless addrs
+      return nil if addrs.empty?
+      thing = addrs.shift
 
-      if cv = email.match(/\A([^@]+)[@]([^@]+)\z/)
+      if cv = thing[:address].match(/\A([^\s]+)[@]([^@]+)\z/) ||
+         cv = thing[:address].match(/\A(["].+?["])[@]([^@]+)\z/)
         # Get the local part and the domain part from the email address
         lpart = cv[1]
         dpart = cv[2]
+        email = Sisimai::Address.expand_verp(thing[:address])
+        aname = nil
 
-        # Remove MIME-Encoded comment part
-        lpart = lpart.sub(/\A=[?].+[?]b[?].+[?]=/, '')
-        lpart = lpart.delete(%q|`'"<>|) unless lpart =~ /\A["].+["]\z/
-        aflag = false
-        addr0 = sprintf('%s@%s', lpart, dpart)
-        addr1 = Sisimai::Address.expand_verp(addr0)
-
-        if addr1.size < 1
-          addr1 = Sisimai::Address.expand_alias(addr0)
-          aflag = true if addr1.size > 0
+        if email.empty?
+          # Is not VERP address, try to expand the address as an alias
+          email = Sisimai::Address.expand_alias(thing[:address])
+          aname = true unless email.empty?
         end
 
-        if addr1.size > 0
-          # The email address is VERP or alias
-          addrs = addr1.split('@')
-          if aflag
-            # The email address is an alias
-            @alias = addr0
-
+        if email =~ /\A.+[@].+?\z/
+          # The address is a VERP or an alias
+          if aname
+            # The address is an alias: neko+nyaan@example.jp
+            @alias = thing[:address]
           else
-            # The email address is a VERP
-            @verp = addr0
+            # The address is a VERP: b+neko=example.jp@example.org
+            @verp  = thing[:address]
           end
-          @user = addrs[0]
-          @host = addrs[1]
-
-        else
-          # The email address is neither VERP nor alias.
-          @user = lpart
-          @host = dpart
-
         end
-        @address = sprintf('%s@%s', @user, @host)
-        @alias ||= ''
-        @verp  ||= ''
+        @user    = lpart
+        @host    = dpart
+        @address = sprintf("%s@%s", lpart, dpart)
 
       else
         # The argument does not include "@"
-        return nil unless Sisimai::RFC5322.is_mailerdaemon(email)
-        @alias ||= ''
-        @verp  ||= ''
-        @host  ||= ''
+        return nil unless Sisimai::RFC5322.is_mailerdaemon(thing[:address])
+        return nil if thing[:address] =~ /[ ]/
 
-        if cv = email.match(/[<]([^ ]+)[>]/)
-          # Mail Delivery Subsystem <MAILER-DAEMON>
-          @user = cv[1]
-          @address = cv[1]
-        else
-          return nil if email =~ /[ ]/
-          # The argument does not include " "
-          @user = email
-          @address = email
-        end
+        # The argument does not include " "
+        @user    = thing[:address]
+        @host  ||= ''
+        @address = thing[:address]
       end
+
+      @alias ||= ''
+      @verp  ||= ''
+      @name    = thing[:name]    || ''
+      @comment = thing[:comment] || ''
     end
 
     # Check whether the object has valid content or not
@@ -479,7 +447,7 @@ module Sisimai
     # Returns the value of address as String
     # @return [String] Email address
     def to_json(*)
-      return to_s
+      return self.address.to_s
     end
 
     # Returns the value of address as String

data/lib/sisimai/data.rb

@@ -61,30 +61,21 @@ module Sisimai
       thing = {}
 
       # Create email address object
-      x0 = Sisimai::Address.find(argvs['addresser'])
-      y0 = Sisimai::Address.find(argvs['recipient'])
-
-      return nil unless x0.is_a? Array
-      return nil unless y0.is_a? Array
-
-      thing[:addresser] = Sisimai::Address.new(x0[0]['address'])
-      return nil unless thing[:addresser].is_a? Sisimai::Address
-      return nil if thing[:addresser].void
-      thing[:senderdomain] = thing[:addresser].host
-
-      thing[:recipient] = Sisimai::Address.new(y0[0]['address'])
-      return nil unless thing[:recipient].is_a? Sisimai::Address
-      return nil if thing[:recipient].void
-      thing[:destination] = thing[:recipient].host
-      thing[:alias] = argvs['alias'] || ''
-
-      @addresser    = thing[:addresser]
-      @senderdomain = thing[:senderdomain]
-      @recipient    = thing[:recipient]
-      @destination  = thing[:destination]
-      @alias        = thing[:alias]
-
-      @token = Sisimai::String.token(@addresser.address, @recipient.address, argvs['timestamp'])
+      as = Sisimai::Address.make(argvs['addresser'])
+      ar = Sisimai::Address.make({ address: argvs['recipient'] })
+
+      return nil unless as.is_a? Sisimai::Address
+      return nil unless ar.is_a? Sisimai::Address
+
+      return nil if as.void
+      return nil if ar.void
+
+      @addresser = as
+      @recipient = ar
+      @senderdomain = as.host
+      @destination  = ar.host
+      @alias = argvs['alias'] || ''
+      @token = Sisimai::String.token(as.address, ar.address, argvs['timestamp'])
       @timestamp = Sisimai::Time.parse(::Time.at(argvs['timestamp']).to_s)
       @timezoneoffset = argvs['timezoneoffset'] || '+0000'
       @lhost          = argvs['lhost']          || ''
@@ -180,17 +171,20 @@ module Sisimai
           # Check each header in message/rfc822 part
           h = f.downcase
           next unless rfc822data.key?(h)
-          next unless rfc822data[h].size > 0
-          next unless Sisimai::RFC5322.is_emailaddress(rfc822data[h])
-          p['addresser'] = rfc822data[h]
+          next if rfc822data[h].empty?
+
+          j = Sisimai::Address.find(rfc822data[h]) || []
+          next if j.empty?
+          p['addresser'] = j[0]
           break
         end
 
-        # Fallback: Get the sender address from the header of the bounced
-        # email if the address is not set at loop above.
-        p['addresser'] ||= ''
-        p['addresser']   = messageobj.header['to'] if p['addresser'].empty?
-
+        unless p['addresser']
+          # Fallback: Get the sender address from the header of the bounced
+          # email if the address is not set at loop above.
+          j = Sisimai::Address.find(messageobj.header['to']) || []
+          p['addresser'] = j[0] unless j.empty?
+        end
         next unless p['addresser']
         next unless p['recipient']
 

data/lib/sisimai/message/email.rb

@@ -302,7 +302,9 @@ module Sisimai
       def self.takeapart(heads)
         return {} unless heads
 
-        # Convert from string to hash reference
+        # 1. Scrub to avoid "invalid byte sequence in UTF-8" exception (#82)
+        # 2. Convert from string to hash reference
+        heads = heads.scrub('?')
         heads = heads.gsub(/^[>]+[ ]/m, '')
 
         takenapart = {}

data/lib/sisimai/rfc3464.rb

@@ -95,7 +95,7 @@ module Sisimai
             end
           end
 
-          if readcursor & Indicators[:'message-rfc822'] > 0
+          if readcursor & Indicators[:'message-rfc822'] == 0
             # Beginning of the original message part
             if e =~ Re1[:rfc822]
               readcursor |= Indicators[:'message-rfc822']
@@ -403,6 +403,7 @@ module Sisimai
               # May be an email address
               x = b['recipient'] || ''
               y = Sisimai::Address.s3s4(cv[1])
+              next unless Sisimai::RFC5322.is_emailaddress(y)
 
               if x.size > 0 && x != y
                 # There are multiple recipient addresses in the message body.
@@ -424,6 +425,26 @@ module Sisimai
           break
         end
 
+        if recipients.zero?
+          # Try to get a recipient address from email headers
+          rfc822list.each do |e|
+            # Check To: header in the original message
+            if cv = e.match(/\ATo:\s*(.+)\z/)
+              r = Sisimai::Address.find(cv[1], true)
+              b = nil
+              next if r.empty?
+
+              if dscontents.size == recipients
+                dscontents << Sisimai::Bite::Email.DELIVERYSTATUS
+              end
+              b = dscontents[-1]
+              b['recipient'] = r[0][:address]
+              b['agent'] = Sisimai::RFC3464.smtpagent + '::Fallback'
+              recipients += 1
+            end
+          end
+        end
+
         return nil unless recipients > 0
         require 'sisimai/string'
         require 'sisimai/smtp/status'

data/lib/sisimai/rfc5322.rb

@@ -38,9 +38,9 @@ module Sisimai
         local_part     = %r/(?:#{dot_atom}|#{quoted_string})/o
         domain         = %r/(?:#{dot_atom}|#{domain_literal})/o
 
-        re[:rfc5322]   = %r/#{local_part}[@]#{domain}/o
-        re[:ignored]   = %r/#{local_part}[.]*[@]#{domain}/o
-        re[:domain]    = %r/#{domain}/o
+        re[:rfc5322]   = %r/\A#{local_part}[@]#{domain}\z/o
+        re[:ignored]   = %r/\A#{local_part}[.]*[@]#{domain}\z/o
+        re[:domain]    = %r/\A#{domain}\z/o
 
         return re
       end
@@ -83,6 +83,7 @@ module Sisimai
       def is_emailaddress(email)
         return false unless email.is_a?(::String)
         return false if email =~ %r/(?:[\x00-\x1f]|\x1f)/
+        return false if email.size > 254
         return true  if email =~ Re[:ignored]
         return false
       end

data/lib/sisimai/rhost/exchangeonline.rb

@@ -116,6 +116,12 @@ module Sisimai
               :regexp => %r/System incorrectly configured/,
             },
           ],
+          %r/\A5[.]4[.]1\z/ => [
+            {
+              :reason => 'userunknown',
+              :regexp => %r/Recipient address rejected: Access denied/,
+            },
+          ],
           %r/\A5[.]4[.][46]\z/ => [
             {
               :reason => 'networkerror',
@@ -234,23 +240,22 @@ module Sisimai
           ],
         }
 
-        # Detect bounce reason from Google Apps
+        # Detect bounce reason from Exchange Online
         # @param    [Sisimai::Data] argvs   Parsed email object
         # @return   [String]                The bounce reason for Exchange Online
-        # @see      https://support.google.com/a/answer/3726730?hl=en
         def get(argvs)
           return nil unless argvs
           return nil unless argvs.is_a? Sisimai::Data
           return argvs.reason if argvs.reason.size > 0
 
-          statuscode = argvs.deliverystatus.sub(/\A\d[.](\d+[.]\d+)\z/, 'X.\1')
+          statuscode = argvs.deliverystatus
           statusmesg = argvs.diagnosticcode
           reasontext = ''
 
           CodeTable.each_key do |e|
             # Try to match with each regular expression of delivery status codes
             next unless statuscode =~ e
-            Codetable[e].each do |f|
+            CodeTable[e].each do |f|
               # Try to match with each regular expression of error messages
               next unless statusmesg =~ f[:regexp]
               reasontext = f[:reason]

data/lib/sisimai/version.rb

@@ -1,4 +1,4 @@
 # Define the version number of Sisimai
 module Sisimai
-  VERSION = '4.22.0'.freeze
+  VERSION = '4.22.1'.freeze
 end

data/set-of-emails/maildir/bsd/email-postfix-30.eml

@@ -0,0 +1,81 @@
+Return-Path: <>
+Received: from [redacted].net.br ([redacted].net.br [[redacted]])
+ by inbound-smtp.us-east-1.amazonaws.com with SMTP id [redacted]
+ for bounces+OkGew2p6d9r5RJ5x9g1nxNq73zQDL1Zo-[redacted].br@example.net.br;
+ Sat, 12 Aug 2017 21:57:09 +0000 (UTC)
+X-SES-Spam-Verdict: PASS
+X-SES-Virus-Verdict: PASS
+Received-SPF: pass (spfCheck: domain of [redacted].net.br designates [redacted] as permitted sender) client-ip=[redacted]; envelope-from=postmaster@example.net.br; helo=[redacted].net.br;
+Authentication-Results: amazonses.com;
+ spf=pass (spfCheck: domain of [redacted].net.br designates [redacted] as permitted sender) client-ip=[redacted]; envelope-from=postmaster@example.net.br; helo=[redacted].net.br;
+X-SES-RECEIPT: [redacted]
+X-SES-DKIM-SIGNATURE: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/simple;
+	s=224i4yxa5dv7c2xz3womw6peuasteono; d=amazonses.com; t=1502575030;
+	h=X-SES-RECEIPT:Date:From:Subject:To:MIME-Version:Content-Type:Message-Id;
+	bh=BWidKjC4S0NbHfJyNpzlk805DzVa5/bMRs6QoN8OTpo=;
+	b=kOtH6OmyLBgqsUExvRp4/yNJTZheGi08a5EMplRGuRxHqoziBHHQjOh0UwqDMIKZ
+	IFzbiAKX4lRuWFi/Ntb8/sijgcPyKqBu269VwAnGmUYEJvAV55D4uXnJjftu+CT7WCr
+	L6SkHjYmTQSWpTwvXmM1YDcUP2dkpWV5+Ko9S4lY=
+Received: by [redacted].net.br (Postfix)
+	id 84FFABD9; Sat, 12 Aug 2017 21:57:09 +0000 (UTC)
+Date: Sat, 12 Aug 2017 21:57:09 +0000 (UTC)
+From: MAILER-DAEMON@example.net.br (Mail Delivery System)
+Subject: Undelivered Mail Returned to Sender
+To: bounces+OkGew2p6d9r5RJ5x9g1nxNq73zQDL1Zo-[redacted].br@example.net.br
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+	boundary="F180CB7A.1502575029/[redacted].net.br"
+Content-Transfer-Encoding: 8bit
+Message-Id: <20170812215709.84FFABD9@example.net.br>
+
+This is a MIME-encapsulated message.
+
+--F180CB7A.1502575029/[redacted].net.br
+Content-Description: Notification
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 8bit
+
+This is the mail system at host [redacted].net.br.
+
+I'm sorry to have to inform you that your message could not
+be delivered to one or more recipients. It's attached below.
+
+For further assistance, please send mail to postmaster.
+
+If you do so, please include this problem report. You can
+delete your own text from the attached returned message.
+
+                   The mail system
+
+<kijitora@example.br>: host
+    [redacted]-br.mail.protection.outlook.com[207.46.163.74] said: 550 5.4.1
+    [kijitora@example.br]: Recipient address rejected: Access denied
+    [BL2NAM02FT061.eop-nam02.prod.protection.outlook.com] (in reply to RCPT TO
+    command)
+
+--F180CB7A.1502575029/[redacted].net.br
+Content-Description: Delivery report
+Content-Type: message/delivery-status
+Content-Transfer-Encoding: 8bit
+
+Reporting-MTA: dns; [redacted].net.br
+X-Postfix-Queue-ID: F180CB7A
+X-Postfix-Sender: rfc822; bounces+OkGew2p6d9r5RJ5x9g1nxNq73zQDL1Zo-[redacted].br@example.net.br
+Arrival-Date: Sat, 12 Aug 2017 21:56:53 +0000 (UTC)
+
+Final-Recipient: rfc822; kijitora@example.br
+Original-Recipient: rfc822;kijitora@example.br
+Action: failed
+Status: 5.4.1
+Remote-MTA: dns; [redacted]-br.mail.protection.outlook.com
+Diagnostic-Code: smtp; 550 5.4.1 [kijitora@example.br]: Recipient
+    address rejected: Access denied
+    [BL2NAM02FT061.eop-nam02.prod.protection.outlook.com]
+
+--F180CB7A.1502575029/[redacted].net.br
+Content-Description: Undelivered Message
+Content-Type: message/rfc822
+Content-Transfer-Encoding: 8bit
+
+[from here, just a copy of the original message, with full headers.]

data/sisimai.gemspec

@@ -24,5 +24,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler', '~> 1.8'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 0'
-  spec.add_runtime_dependency 'oj', '~> 2.14', '>= 2.14.4'
+  spec.add_runtime_dependency 'oj', '~> 3.0.0', '>= 3.0.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sisimai
 version: !ruby/object:Gem::Version
-  version: 4.22.0
+  version: 4.22.1
 platform: ruby
 authors:
 - azumakuniyuki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-08-22 00:00:00.000000000 Z
+date: 2017-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,20 +58,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.14'
+        version: 3.0.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.14.4
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.14'
+        version: 3.0.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.14.4
+        version: 3.0.0
 description: Sisimai is a Ruby library for analyzing RFC5322 bounce emails and generating
   structured data from parsed results.
 email:
@@ -453,6 +453,7 @@ files:
 - set-of-emails/maildir/bsd/email-postfix-27.eml
 - set-of-emails/maildir/bsd/email-postfix-28.eml
 - set-of-emails/maildir/bsd/email-postfix-29.eml
+- set-of-emails/maildir/bsd/email-postfix-30.eml
 - set-of-emails/maildir/bsd/email-qmail-01.eml
 - set-of-emails/maildir/bsd/email-qmail-02.eml
 - set-of-emails/maildir/bsd/email-qmail-03.eml