net-imap 0.4.2 → 0.4.4

data/lib/net/imap/response_parser.rb

@@ -58,6 +58,21 @@ module Net
       T_TEXT     = :TEXT         # any char except CRLF
       T_EOF      = :EOF          # end of response string
 
+      module ResponseConditions
+        OK      = "OK"
+        NO      = "NO"
+        BAD     = "BAD"
+        BYE     = "BYE"
+        PREAUTH = "PREAUTH"
+
+        RESP_COND_STATES      = [OK, NO, BAD              ].freeze
+        RESP_DATA_CONDS       = [OK, NO, BAD, BYE,        ].freeze
+        AUTH_CONDS            = [OK,               PREAUTH].freeze
+        GREETING_CONDS        = [OK,          BYE, PREAUTH].freeze
+        RESP_CONDS            = [OK, NO, BAD, BYE, PREAUTH].freeze
+      end
+      include ResponseConditions
+
       module Patterns
 
         module CharClassSubtraction
@@ -170,6 +185,54 @@ module Net
         CODE_TEXT_CHAR    = TEXT_CHAR - RESP_SPECIALS
         CODE_TEXT         = /#{CODE_TEXT_CHAR}+/n
 
+        # flag            = "\Answered" / "\Flagged" / "\Deleted" /
+        #                   "\Seen" / "\Draft" / flag-keyword / flag-extension
+        #                     ; Does not include "\Recent"
+        # flag-extension  = "\" atom
+        #                     ; Future expansion.  Client implementations
+        #                     ; MUST accept flag-extension flags.  Server
+        #                     ; implementations MUST NOT generate
+        #                     ; flag-extension flags except as defined by
+        #                     ; a future Standard or Standards Track
+        #                     ; revisions of this specification.
+        # flag-keyword    = "$MDNSent" / "$Forwarded" / "$Junk" /
+        #                   "$NotJunk" / "$Phishing" / atom
+        # flag-perm       = flag / "\*"
+        #
+        # Not checking for max one mbx-list-sflag in the parser.
+        # >>>
+        # mbx-list-oflag  = "\Noinferiors" / child-mbox-flag /
+        #                   "\Subscribed" / "\Remote" / flag-extension
+        #                    ; Other flags; multiple from this list are
+        #                    ; possible per LIST response, but each flag
+        #                    ; can only appear once per LIST response
+        # mbx-list-sflag  = "\NonExistent" / "\Noselect" / "\Marked" /
+        #                   "\Unmarked"
+        #                    ; Selectability flags; only one per LIST response
+        # child-mbox-flag =  "\HasChildren" / "\HasNoChildren"
+        #                    ; attributes for the CHILDREN return option, at most
+        #                    ; one possible per LIST response
+        FLAG              = /\\?#{ATOM}/n
+        FLAG_EXTENSION    = /\\#{ATOM}/n
+        FLAG_KEYWORD      = ATOM
+        FLAG_PERM         = Regexp.union(FLAG, "\\*")
+        MBX_FLAG          = FLAG_EXTENSION
+
+        # flag-list       = "(" [flag *(SP flag)] ")"
+        #
+        # part of resp-text-code:
+        # >>>
+        #   "PERMANENTFLAGS" SP "(" [flag-perm *(SP flag-perm)] ")"
+        #
+        # parens from mailbox-list are included in the regexp:
+        # >>>
+        #   mbx-list-flags  = *(mbx-list-oflag SP) mbx-list-sflag
+        #                     *(SP mbx-list-oflag) /
+        #                     mbx-list-oflag *(SP mbx-list-oflag)
+        FLAG_LIST      = /\G\((#{FLAG     }(?:#{SP}#{FLAG     })*|)\)/ni
+        FLAG_PERM_LIST = /\G\((#{FLAG_PERM}(?:#{SP}#{FLAG_PERM})*|)\)/ni
+        MBX_LIST_FLAGS = /\G\((#{MBX_FLAG }(?:#{SP}#{MBX_FLAG })*|)\)/ni
+
         # RFC3501:
         #   QUOTED-CHAR   = <any TEXT-CHAR except quoted-specials> /
         #                   "\" quoted-specials
@@ -195,6 +258,14 @@ module Net
         TEXT_rev1         = /#{TEXT_CHAR}+/
         TEXT_rev2         = /#{Regexp.union TEXT_CHAR, UTF8_2, UTF8_3, UTF8_4}+/
 
+        # tagged-label-fchar = ALPHA / "-" / "_" / "."
+        TAGGED_LABEL_FCHAR   = /[a-zA-Z\-_.]/n
+        # tagged-label-char  = tagged-label-fchar / DIGIT / ":"
+        TAGGED_LABEL_CHAR    = /[a-zA-Z\-_.0-9:]*/n
+        # tagged-ext-label   = tagged-label-fchar *tagged-label-char
+        #                      ; Is a valid RFC 3501 "atom".
+        TAGGED_EXT_LABEL     = /#{TAGGED_LABEL_FCHAR}#{TAGGED_LABEL_CHAR}*/n
+
         # RFC3501:
         #   literal          = "{" number "}" CRLF *CHAR8
         #                        ; Number represents the number of CHAR8s
@@ -268,6 +339,8 @@ module Net
       Token = Struct.new(:symbol, :value)
 
       def_char_matchers :SP,   " ", :T_SPACE
+      def_char_matchers :PLUS, "+", :T_PLUS
+      def_char_matchers :STAR, "*", :T_STAR
 
       def_char_matchers :lpar, "(", :T_LPAR
       def_char_matchers :rpar, ")", :T_RPAR
@@ -310,6 +383,9 @@ module Net
       # TODO: add to lexer and only match tagged-ext-label
       def_token_matchers :tagged_ext_label, T_ATOM, T_NIL, send: :upcase
 
+      def_token_matchers :CRLF, T_CRLF
+      def_token_matchers :EOF,  T_EOF
+
       # atom            = 1*ATOM-CHAR
       # ATOM-CHAR       = <any CHAR except atom-specials>
       ATOM_TOKENS = [T_ATOM, T_NUMBER, T_NIL, T_LBRA, T_PLUS]
@@ -320,10 +396,13 @@ module Net
 
       ASTRING_TOKENS = [T_QUOTED, *ASTRING_CHARS_TOKENS, T_LITERAL].freeze
 
-      # atom            = 1*ATOM-CHAR
-      #
-      # TODO: match atom entirely by regexp (in the "lexer")
-      def atom; -combine_adjacent(*ATOM_TOKENS) end
+      # tag             = 1*<any ASTRING-CHAR except "+">
+      TAG_TOKENS = (ASTRING_CHARS_TOKENS - [T_PLUS]).freeze
+
+      # TODO: handle atom, astring_chars, and tag entirely inside the lexer
+      def atom;          combine_adjacent(*ATOM_TOKENS)          end
+      def astring_chars; combine_adjacent(*ASTRING_CHARS_TOKENS) end
+      def tag;           combine_adjacent(*TAG_TOKENS)           end
 
       # the #accept version of #atom
       def atom?; -combine_adjacent(*ATOM_TOKENS) if lookahead?(*ATOM_TOKENS) end
@@ -336,11 +415,6 @@ module Net
         -combine_adjacent(*ATOM_TOKENS).upcase if lookahead?(*ATOM_TOKENS)
       end
 
-      # TODO: handle astring_chars entirely inside the lexer
-      def astring_chars
-        combine_adjacent(*ASTRING_CHARS_TOKENS)
-      end
-
       #   astring         = 1*ASTRING-CHAR / string
       def astring
         lookahead?(*ASTRING_CHARS_TOKENS) ? astring_chars : string
@@ -357,6 +431,30 @@ module Net
         parse_error("unexpected atom %p, expected %p instead", val, word)
       end
 
+      # Use #label or #label_in to assert specific known labels
+      # (+tagged-ext-label+ only, not +atom+).
+      def label_in(*labels)
+        lbl = tagged_ext_label and labels.include?(lbl) and return lbl
+        parse_error("unexpected atom %p, expected one of %s instead",
+                    lbl, labels.join(" or "))
+      end
+
+      # expects "OK" or "PREAUTH" and raises InvalidResponseError on failure
+      def resp_cond_auth__name
+        lbl = tagged_ext_label and AUTH_CONDS.include? lbl and return lbl
+        raise InvalidResponseError, "bad response type %p, expected %s" % [
+          lbl, AUTH_CONDS.join(" or ")
+        ]
+      end
+
+      # expects "OK" or "NO" or "BAD" and raises InvalidResponseError on failure
+      def resp_cond_state__name
+        lbl = tagged_ext_label and RESP_COND_STATES.include? lbl and return lbl
+        raise InvalidResponseError, "bad response type %p, expected %s" % [
+          lbl, RESP_COND_STATES.join(" or ")
+        ]
+      end
+
       #   nstring         = string / nil
       def nstring
         NIL? ? nil : string
@@ -378,155 +476,295 @@ module Net
       alias number64    number
       alias number64?   number?
 
-      def response
-        token = lookahead
-        case token.symbol
-        when T_PLUS
-          result = continue_req
-        when T_STAR
-          result = response_untagged
-        else
-          result = response_tagged
-        end
-        while lookahead.symbol == T_SPACE
-          # Ignore trailing space for Microsoft Exchange Server
-          shift_token
-        end
-        match(T_CRLF)
-        match(T_EOF)
-        return result
-      end
+      # valid number ranges are not enforced by parser
+      #   nz-number       = digit-nz *DIGIT
+      #                       ; Non-zero unsigned 32-bit integer
+      #                       ; (0 < n < 4,294,967,296)
+      alias nz_number   number
+      alias nz_number?  number?
+
+      # valid number ranges are not enforced by parser
+      #   nz-number64     = digit-nz *DIGIT
+      #                       ; Unsigned 63-bit integer
+      #                       ; (0 < n <= 9,223,372,036,854,775,807)
+      alias nz_number64 nz_number
 
+      # valid number ranges are not enforced by parser
+      #      uniqueid        = nz-number
+      #                          ; Strictly ascending
+      alias uniqueid    nz_number
+
+      # [RFC3501 & RFC9051:]
+      #   response        = *(continue-req / response-data) response-done
+      #
+      # For simplicity, response isn't interpreted as the combination of the
+      # three response types, but instead represents any individual server
+      # response.  Our simplified interpretation is defined as:
+      #   response        = continue-req | response_data | response-tagged
+      #
+      # n.b: our "response-tagged" definition parses "greeting" too.
+      def response
+        resp = case lookahead!(T_PLUS, T_STAR, *TAG_TOKENS).symbol
+               when T_PLUS then continue_req
+               when T_STAR then response_data
+               else             response_tagged
+               end
+        accept_spaces # QUIRKY: Ignore trailing space (MS Exchange Server?)
+        CRLF!
+        EOF!
+        resp
+      end
+
+      # RFC3501 & RFC9051:
+      #   continue-req    = "+" SP (resp-text / base64) CRLF
+      #
+      # n.b: base64 is valid resp-text.  And in the spirit of RFC9051 Appx E 23
+      # (and to workaround existing servers), we use the following grammar:
+      #
+      #   continue-req    = "+" (SP (resp-text)) CRLF
       def continue_req
-        match(T_PLUS)
-        token = lookahead
-        if token.symbol == T_SPACE
-          shift_token
-          return ContinuationRequest.new(resp_text, @str)
-        else
-          return ContinuationRequest.new(ResponseText.new(nil, ""), @str)
+        PLUS!
+        ContinuationRequest.new(SP? ? resp_text : ResponseText::EMPTY, @str)
+      end
+
+      RE_RESPONSE_TYPE = /\G(?:\d+ )?(?<type>#{Patterns::TAGGED_EXT_LABEL})/n
+
+      # [RFC3501:]
+      #   response-data    = "*" SP (resp-cond-state / resp-cond-bye /
+      #                      mailbox-data / message-data / capability-data) CRLF
+      # [RFC4466:]
+      #   response-data    = "*" SP response-payload CRLF
+      #   response-payload = resp-cond-state / resp-cond-bye /
+      #                       mailbox-data / message-data / capability-data
+      # RFC5161 (ENABLE capability):
+      #   response-data    =/ "*" SP enable-data CRLF
+      # RFC5255 (LANGUAGE capability)
+      #   response-payload =/ language-data
+      # RFC5255 (I18NLEVEL=1 and I18NLEVEL=2 capabilities)
+      #   response-payload =/ comparator-data
+      # [RFC9051:]
+      #   response-data    = "*" SP (resp-cond-state / resp-cond-bye /
+      #                      mailbox-data / message-data / capability-data /
+      #                      enable-data) CRLF
+      #
+      # [merging in greeting and response-fatal:]
+      #   greeting         =  "*" SP (resp-cond-auth / resp-cond-bye) CRLF
+      #   response-fatal   =  "*" SP resp-cond-bye CRLF
+      #   response-data    =/ "*" SP (resp-cond-auth / resp-cond-bye) CRLF
+      # [removing duplicates, this is simply]
+      #   response-payload =/ resp-cond-auth
+      #
+      # TODO: remove resp-cond-auth and handle greeting separately
+      def response_data
+        STAR!; SP!
+        m = peek_re(RE_RESPONSE_TYPE) or parse_error("unparsable response")
+        case m["type"].upcase
+        when "OK"         then resp_cond_state__untagged # RFC3501, RFC9051
+        when "FETCH"      then message_data__fetch       # RFC3501, RFC9051
+        when "EXPUNGE"    then message_data__expunge     # RFC3501, RFC9051
+        when "EXISTS"     then mailbox_data__exists      # RFC3501, RFC9051
+        when "ESEARCH"    then esearch_response          # RFC4731, RFC9051, etc
+        when "VANISHED"   then expunged_resp             # RFC7162
+        when "UIDFETCH"   then uidfetch_resp             # (draft) UIDONLY
+        when "SEARCH"     then mailbox_data__search      # RFC3501 (obsolete)
+        when "CAPABILITY" then capability_data__untagged # RFC3501, RFC9051
+        when "FLAGS"      then mailbox_data__flags       # RFC3501, RFC9051
+        when "LIST"       then mailbox_data__list        # RFC3501, RFC9051
+        when "STATUS"     then mailbox_data__status      # RFC3501, RFC9051
+        when "NAMESPACE"  then namespace_response        # RFC2342, RFC9051
+        when "ENABLED"    then enable_data               # RFC5161, RFC9051
+        when "BAD"        then resp_cond_state__untagged # RFC3501, RFC9051
+        when "NO"         then resp_cond_state__untagged # RFC3501, RFC9051
+        when "PREAUTH"    then resp_cond_auth            # RFC3501, RFC9051
+        when "BYE"        then resp_cond_bye             # RFC3501, RFC9051
+        when "RECENT"     then mailbox_data__recent      # RFC3501 (obsolete)
+        when "SORT"       then sort_data                 # RFC5256, RFC7162
+        when "THREAD"     then thread_data               # RFC5256
+        when "QUOTA"      then quota_response            # RFC2087, RFC9208
+        when "QUOTAROOT"  then quotaroot_response        # RFC2087, RFC9208
+        when "ID"         then id_response               # RFC2971
+        when "ACL"        then acl_data                  # RFC4314
+        when "LISTRIGHTS" then listrights_data           # RFC4314
+        when "MYRIGHTS"   then myrights_data             # RFC4314
+        when "METADATA"   then metadata_resp             # RFC5464
+        when "LANGUAGE"   then language_data             # RFC5255
+        when "COMPARATOR" then comparator_data           # RFC5255
+        when "CONVERTED"  then message_data__converted   # RFC5259
+        when "LSUB"       then mailbox_data__lsub        # RFC3501 (obsolete)
+        when "XLIST"      then mailbox_data__xlist       # deprecated
+        when "NOOP"       then response_data__noop
+        else                   response_data__unhandled
         end
       end
 
-      def response_untagged
-        match(T_STAR)
-        match(T_SPACE)
-        token = lookahead
-        if token.symbol == T_NUMBER
-          return numeric_response
-        elsif token.symbol == T_ATOM
-          case token.value
-          when /\A(?:OK|NO|BAD|BYE|PREAUTH)\z/ni
-            return response_cond
-          when /\A(?:FLAGS)\z/ni
-            return flags_response
-          when /\A(?:ID)\z/ni
-            return id_response
-          when /\A(?:LIST|LSUB|XLIST)\z/ni
-            return list_response
-          when /\A(?:NAMESPACE)\z/ni
-            return namespace_response
-          when /\A(?:QUOTA)\z/ni
-            return getquota_response
-          when /\A(?:QUOTAROOT)\z/ni
-            return getquotaroot_response
-          when /\A(?:ACL)\z/ni
-            return getacl_response
-          when /\A(?:SEARCH|SORT)\z/ni
-            return search_response
-          when /\A(?:THREAD)\z/ni
-            return thread_response
-          when /\A(?:STATUS)\z/ni
-            return status_response
-          when /\A(?:CAPABILITY)\z/ni
-            return capability_data__untagged
-          when /\A(?:NOOP)\z/ni
-            return ignored_response
-          when /\A(?:ENABLED)\z/ni
-            return enable_data
-          else
-            return text_response
+      def response_data__unhandled(klass = UntaggedResponse)
+        num  = number?;          SP?
+        type = tagged_ext_label; SP?
+        text = remaining_unparsed
+        data =
+          if num && text then UnparsedNumericResponseData.new(num, text)
+          elsif     text then UnparsedData.new(text)
+          else                num
           end
-        else
-          parse_error("unexpected token %s", token.symbol)
-        end
+        klass.new(type, data, @str)
+      end
+
+      # reads all the way up until CRLF
+      def remaining_unparsed
+        str = @str[@pos...-2] and @pos += str.bytesize
+        str&.empty? ? nil : str
       end
 
+      def response_data__ignored; response_data__unhandled(IgnoredResponse) end
+      alias response_data__noop     response_data__ignored
+
+      alias esearch_response        response_data__unhandled
+      alias expunged_resp           response_data__unhandled
+      alias uidfetch_resp           response_data__unhandled
+      alias listrights_data         response_data__unhandled
+      alias myrights_data           response_data__unhandled
+      alias metadata_resp           response_data__unhandled
+      alias language_data           response_data__unhandled
+      alias comparator_data         response_data__unhandled
+      alias message_data__converted response_data__unhandled
+
+      # RFC3501 & RFC9051:
+      #   response-tagged = tag SP resp-cond-state CRLF
+      #
+      #   resp-cond-state = ("OK" / "NO" / "BAD") SP resp-text
+      #                       ; Status condition
+      #
+      #   tag             = 1*<any ASTRING-CHAR except "+">
       def response_tagged
-        tag = astring_chars
-        match(T_SPACE)
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        return TaggedResponse.new(tag, name, resp_text, @str)
+        tag  = tag();                 SP!
+        name = resp_cond_state__name; SP!
+        TaggedResponse.new(tag, name, resp_text, @str)
       end
 
-      def response_cond
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        return UntaggedResponse.new(name, resp_text, @str)
+      # RFC3501 & RFC9051:
+      #   resp-cond-state  = ("OK" / "NO" / "BAD") SP resp-text
+      def resp_cond_state__untagged
+        name = resp_cond_state__name; SP!
+        UntaggedResponse.new(name, resp_text, @str)
       end
 
-      def numeric_response
-        n = number
-        match(T_SPACE)
-        token = match(T_ATOM)
-        name = token.value.upcase
-        case name
-        when "EXISTS", "RECENT", "EXPUNGE"
-          return UntaggedResponse.new(name, n, @str)
-        when "FETCH"
-          shift_token
-          match(T_SPACE)
-          data = FetchData.new(n, msg_att(n))
-          return UntaggedResponse.new(name, data, @str)
-        end
+      #   resp-cond-auth   = ("OK" / "PREAUTH") SP resp-text
+      def resp_cond_auth
+        name = resp_cond_auth__name; SP!
+        UntaggedResponse.new(name, resp_text, @str)
       end
 
+      #   resp-cond-bye    = "BYE" SP resp-text
+      def resp_cond_bye
+        name = label(BYE); SP!
+        UntaggedResponse.new(name, resp_text, @str)
+      end
+
+      #   message-data    = nz-number SP ("EXPUNGE" / ("FETCH" SP msg-att))
+      def message_data__fetch
+        seq  = nz_number;     SP!
+        name = label "FETCH"; SP!
+        data = FetchData.new(seq, msg_att(seq))
+        UntaggedResponse.new(name, data, @str)
+      end
+
+      def response_data__simple_numeric
+        data = nz_number; SP!
+        name = tagged_ext_label
+        UntaggedResponse.new(name, data, @str)
+      end
+
+      alias message_data__expunge response_data__simple_numeric
+      alias mailbox_data__exists  response_data__simple_numeric
+      alias mailbox_data__recent  response_data__simple_numeric
+
+      # RFC3501 & RFC9051:
+      #   msg-att         = "(" (msg-att-dynamic / msg-att-static)
+      #                      *(SP (msg-att-dynamic / msg-att-static)) ")"
+      #
+      #   msg-att-dynamic = "FLAGS" SP "(" [flag-fetch *(SP flag-fetch)] ")"
+      # RFC5257 (ANNOTATE extension):
+      #   msg-att-dynamic =/ "ANNOTATION" SP
+      #                        ( "(" entry-att *(SP entry-att) ")" /
+      #                          "(" entry *(SP entry) ")" )
+      # RFC7162 (CONDSTORE extension):
+      #   msg-att-dynamic =/ fetch-mod-resp
+      #   fetch-mod-resp  = "MODSEQ" SP "(" permsg-modsequence ")"
+      # RFC8970 (PREVIEW extension):
+      #   msg-att-dynamic =/ "PREVIEW" SP nstring
+      #
+      # RFC3501:
+      #   msg-att-static  = "ENVELOPE" SP envelope /
+      #                     "INTERNALDATE" SP date-time /
+      #                     "RFC822" [".HEADER" / ".TEXT"] SP nstring /
+      #                     "RFC822.SIZE" SP number /
+      #                     "BODY" ["STRUCTURE"] SP body /
+      #                     "BODY" section ["<" number ">"] SP nstring /
+      #                     "UID" SP uniqueid
+      # RFC3516 (BINARY extension):
+      #   msg-att-static  =/ "BINARY" section-binary SP (nstring / literal8)
+      #                    / "BINARY.SIZE" section-binary SP number
+      # RFC8514 (SAVEDATE extension):
+      #   msg-att-static  =/ "SAVEDATE" SP (date-time / nil)
+      # RFC8474 (OBJECTID extension):
+      #   msg-att-static =/ fetch-emailid-resp / fetch-threadid-resp
+      #   fetch-emailid-resp  = "EMAILID" SP "(" objectid ")"
+      #   fetch-threadid-resp = "THREADID" SP ( "(" objectid ")" / nil )
+      # RFC9051:
+      #   msg-att-static  = "ENVELOPE" SP envelope /
+      #                     "INTERNALDATE" SP date-time /
+      #                     "RFC822.SIZE" SP number64 /
+      #                     "BODY" ["STRUCTURE"] SP body /
+      #                     "BODY" section ["<" number ">"] SP nstring /
+      #                     "BINARY" section-binary SP (nstring / literal8) /
+      #                     "BINARY.SIZE" section-binary SP number /
+      #                     "UID" SP uniqueid
+      #
+      # Re https://www.rfc-editor.org/errata/eid7246, I'm adding "offset" to the
+      # official "BINARY" ABNF, like so:
+      #
+      #   msg-att-static   =/ "BINARY" section-binary ["<" number ">"] SP
+      #                       (nstring / literal8)
       def msg_att(n)
-        match(T_LPAR)
+        lpar
         attr = {}
         while true
-          token = lookahead
-          case token.symbol
-          when T_RPAR
-            shift_token
-            break
-          when T_SPACE
-            shift_token
-            next
-          end
-          case token.value
-          when /\A(?:ENVELOPE)\z/ni
-            name, val = envelope_data
-          when /\A(?:FLAGS)\z/ni
-            name, val = flags_data
-          when /\A(?:INTERNALDATE)\z/ni
-            name, val = internaldate_data
-          when /\A(?:RFC822(?:\.HEADER|\.TEXT)?)\z/ni
-            name, val = rfc822_text
-          when /\A(?:RFC822\.SIZE)\z/ni
-            name, val = rfc822_size
-          when /\A(?:BODY(?:STRUCTURE)?)\z/ni
-            name, val = body_data
-          when /\A(?:UID)\z/ni
-            name, val = uid_data
-          when /\A(?:MODSEQ)\z/ni
-            name, val = modseq_data
-          else
-            parse_error("unknown attribute `%s' for {%d}", token.value, n)
-          end
+          name = msg_att__label; SP!
+          val =
+            case name
+            when "UID"                  then uniqueid
+            when "FLAGS"                then flag_list
+            when "BODY"                 then body
+            when /\ABODY\[/ni           then nstring
+            when "BODYSTRUCTURE"        then body
+            when "ENVELOPE"             then envelope
+            when "INTERNALDATE"         then date_time
+            when "RFC822.SIZE"          then number64
+            when "RFC822"               then nstring            # not in rev2
+            when "RFC822.HEADER"        then nstring            # not in rev2
+            when "RFC822.TEXT"          then nstring            # not in rev2
+            when "MODSEQ"               then parens__modseq     # CONDSTORE
+            else parse_error("unknown attribute `%s' for {%d}", name, n)
+            end
           attr[name] = val
+          break unless SP?
+          break if lookahead_rpar?
         end
-        return attr
-      end
-
-      def envelope_data
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        return name, envelope
+        rpar
+        attr
+      end
+
+      # appends "[section]" and "<partial>" to the base label
+      def msg_att__label
+        case (name = tagged_ext_label)
+        when /\A(?:RFC822(?:\.HEADER|\.TEXT)?)\z/ni
+          # ignoring "[]" fixes https://bugs.ruby-lang.org/issues/5620
+          lbra? and rbra
+        when "BODY"
+          peek_lbra? and name << section and
+            peek_str?("<") and name << atom # partial
+        end
+        name
       end
 
       def envelope
@@ -564,58 +802,10 @@ module Net
         return result
       end
 
-      def flags_data
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        return name, flag_list
-      end
-
-      def internaldate_data
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        token = match(T_QUOTED)
-        return name, token.value
-      end
-
-      def rfc822_text
-        token = match(T_ATOM)
-        name = token.value.upcase
-        token = lookahead
-        if token.symbol == T_LBRA
-          shift_token
-          match(T_RBRA)
-        end
-        match(T_SPACE)
-        return name, nstring
-      end
-
-      def rfc822_size
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        return name, number
-      end
-
-      def body_data
-        token = match(T_ATOM)
-        name = token.value.upcase
-        token = lookahead
-        if token.symbol == T_SPACE
-          shift_token
-          return name, body
-        end
-        name.concat(section)
-        token = lookahead
-        if token.symbol == T_ATOM
-          name.concat(token.value)
-          shift_token
-        end
-        match(T_SPACE)
-        data = nstring
-        return name, data
-      end
+      #   date-time       = DQUOTE date-day-fixed "-" date-month "-" date-year
+      #                     SP time SP zone DQUOTE
+      alias date_time quoted
+      alias ndatetime nquoted
 
       # RFC-3501 & RFC-9051:
       #   body            = "(" (body-type-1part / body-type-mpart) ")"
@@ -844,6 +1034,7 @@ module Net
         if lpar?
           result = [case_insensitive__string]
           result << case_insensitive__string while SP?
+          rpar
           result
         else
           case_insensitive__nstring
@@ -872,101 +1063,90 @@ module Net
         end
       end
 
+      # section         = "[" [section-spec] "]"
       def section
-        str = String.new
-        token = match(T_LBRA)
-        str.concat(token.value)
-        token = match(T_ATOM, T_NUMBER, T_RBRA)
-        if token.symbol == T_RBRA
-          str.concat(token.value)
-          return str
-        end
-        str.concat(token.value)
-        token = lookahead
-        if token.symbol == T_SPACE
-          shift_token
-          str.concat(token.value)
-          token = match(T_LPAR)
-          str.concat(token.value)
-          while true
-            token = lookahead
-            case token.symbol
-            when T_RPAR
-              str.concat(token.value)
-              shift_token
-              break
-            when T_SPACE
-              shift_token
-              str.concat(token.value)
-            end
-            str.concat(format_string(astring))
-          end
-        end
-        token = match(T_RBRA)
-        str.concat(token.value)
-        return str
-      end
-
-      def format_string(str)
-        case str
-        when ""
-          return '""'
-        when /[\x80-\xff\r\n]/n
-          # literal
-          return "{" + str.bytesize.to_s + "}" + CRLF + str
-        when /[(){ \x00-\x1f\x7f%*"\\]/n
-          # quoted string
-          return '"' + str.gsub(/["\\]/n, "\\\\\\&") + '"'
-        else
-          # atom
-          return str
-        end
-      end
-
-      def uid_data
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        return name, number
-      end
-
-      def modseq_data
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        match(T_LPAR)
-        modseq = number
-        match(T_RPAR)
-        return name, modseq
-      end
-
-      def ignored_response
-        while lookahead.symbol != T_CRLF
-          shift_token
-        end
-        return IgnoredResponse.new(@str)
+        str = +lbra
+        str << section_spec unless peek_rbra?
+        str << rbra
+      end
+
+      # section-spec    = section-msgtext / (section-part ["." section-text])
+      # section-msgtext = "HEADER" /
+      #                   "HEADER.FIELDS" [".NOT"] SP header-list /
+      #                   "TEXT"
+      #                     ; top-level or MESSAGE/RFC822 or
+      #                     ; MESSAGE/GLOBAL part
+      # section-part    = nz-number *("." nz-number)
+      #                     ; body part reference.
+      #                     ; Allows for accessing nested body parts.
+      # section-text    = section-msgtext / "MIME"
+      #                     ; text other than actual body part (headers,
+      #                     ; etc.)
+      #
+      # n.b: we could "cheat" here and just grab all text inside the brackets,
+      # but literals would need special treatment.
+      def section_spec
+        str = "".b
+        str << atom # grabs everything up to "SP header-list" or "]"
+        str << " " << header_list if SP?
+        str
       end
 
-      def text_response
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        return UntaggedResponse.new(name, text)
+      # header-list     = "(" header-fld-name *(SP header-fld-name) ")"
+      def header_list
+        str = +""
+        str << lpar << header_fld_name
+        str << " "  << header_fld_name while SP?
+        str << rpar
       end
 
-      def flags_response
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        return UntaggedResponse.new(name, flag_list, @str)
-      end
-
-      def list_response
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        return UntaggedResponse.new(name, mailbox_list, @str)
-      end
+      # RFC3501 & RFC9051:
+      #   header-fld-name = astring
+      #
+      # NOTE: Previously, Net::IMAP recreated the raw original source string.
+      # Now, it grabs the raw encoded value using @str and @pos.  A future
+      # version may simply return the decoded astring value.  Although that is
+      # technically incompatible, it should almost never make a difference: all
+      # standard header field names are valid atoms:
+      #
+      # https://www.iana.org/assignments/message-headers/message-headers.xhtml
+      #
+      # Although RFC3501 allows any astring, RFC5322-valid header names are one
+      # or more of the printable US-ASCII characters, except SP and colon.  So
+      # empty string isn't valid, and literals aren't needed and should not be
+      # used.  This is explicitly unchanged by [I18N-HDRS] (RFC6532).
+      #
+      # RFC5233:
+      #     optional-field  =   field-name ":" unstructured CRLF
+      #     field-name      =   1*ftext
+      #     ftext           =   %d33-57 /          ; Printable US-ASCII
+      #                         %d59-126           ;  characters not including
+      #                                            ;  ":".
+      def header_fld_name
+        assert_no_lookahead
+        start = @pos
+        astring
+        @str[start...@pos - 1]
+      end
+
+      # mailbox-data    =  "FLAGS" SP flag-list / "LIST" SP mailbox-list /
+      #                    "LSUB" SP mailbox-list / "SEARCH" *(SP nz-number) /
+      #                    "STATUS" SP mailbox SP "(" [status-att-list] ")" /
+      #                    number SP "EXISTS" / number SP "RECENT"
+
+      def mailbox_data__flags
+        name = label("FLAGS")
+        SP!
+        UntaggedResponse.new(name, flag_list, @str)
+      end
+
+      def mailbox_data__list
+        name = label_in("LIST", "LSUB", "XLIST")
+        SP!
+        UntaggedResponse.new(name, mailbox_list, @str)
+      end
+      alias mailbox_data__lsub  mailbox_data__list
+      alias mailbox_data__xlist mailbox_data__list
 
       def mailbox_list
         attr = flag_list
@@ -1032,7 +1212,8 @@ module Net
         return UntaggedResponse.new(name, data, @str)
       end
 
-      def getacl_response
+      # acl-data        = "ACL" SP mailbox *(SP identifier SP rights)
+      def acl_data
         token = match(T_ATOM)
         name = token.value.upcase
         match(T_SPACE)
@@ -1058,7 +1239,21 @@ module Net
         return UntaggedResponse.new(name, data, @str)
       end
 
-      def search_response
+      # RFC3501:
+      #   mailbox-data        = "SEARCH" *(SP nz-number) / ...
+      # RFC5256: SORT
+      #   sort-data           = "SORT" *(SP nz-number)
+      # RFC7162: CONDSTORE, QRESYNC
+      #   mailbox-data        =/ "SEARCH" [1*(SP nz-number) SP
+      #                          search-sort-mod-seq]
+      #   sort-data           = "SORT" [1*(SP nz-number) SP
+      #                           search-sort-mod-seq]
+      #                           ; Updates the SORT response from RFC 5256.
+      #   search-sort-mod-seq = "(" "MODSEQ" SP mod-sequence-value ")"
+      # RFC9051:
+      #   mailbox-data        = obsolete-search-response / ...
+      #   obsolete-search-response = "SEARCH" *(SP nz-number)
+      def mailbox_data__search
         token = match(T_ATOM)
         name = token.value.upcase
         token = lookahead
@@ -1088,8 +1283,9 @@ module Net
         end
         return UntaggedResponse.new(name, data, @str)
       end
+      alias sort_data mailbox_data__search
 
-      def thread_response
+      def thread_data
         token = match(T_ATOM)
         name = token.value.upcase
         token = lookahead
@@ -1151,7 +1347,7 @@ module Net
         return rootmember
       end
 
-      def status_response
+      def mailbox_data__status
         token = match(T_ATOM)
         name = token.value.upcase
         match(T_SPACE)
@@ -1198,11 +1394,13 @@ module Net
       end
 
       # As a workaround for buggy servers, allow a trailing SP:
-      #     *(SP capapility) [SP]
+      #     *(SP capability) [SP]
       def capability__list
-        data = []; while _ = SP? && capability? do data << _ end; data
+        list = []; while SP? && (capa = capability?) do list << capa end; list
       end
 
+      alias resp_code__capability capability__list
+
       # capability      = ("AUTH=" auth-type) / atom
       #                     ; New capabilities MUST begin with "X" or be
       #                     ; registered with IANA as standard or
@@ -1325,68 +1523,91 @@ module Net
         end
       end
 
-      # See https://www.rfc-editor.org/errata/rfc3501
+      # RFC3501 (See https://www.rfc-editor.org/errata/rfc3501):
+      #   resp-text-code   = "ALERT" /
+      #                      "BADCHARSET" [SP "(" charset *(SP charset) ")" ] /
+      #                      capability-data / "PARSE" /
+      #                      "PERMANENTFLAGS" SP "(" [flag-perm *(SP flag-perm)] ")" /
+      #                      "READ-ONLY" / "READ-WRITE" / "TRYCREATE" /
+      #                      "UIDNEXT" SP nz-number / "UIDVALIDITY" SP nz-number /
+      #                      "UNSEEN" SP nz-number /
+      #                      atom [SP 1*<any TEXT-CHAR except "]">]
+      #   capability-data  = "CAPABILITY" *(SP capability) SP "IMAP4rev1"
+      #                      *(SP capability)
       #
-      # resp-text-code  = "ALERT" /
-      #                   "BADCHARSET" [SP "(" charset *(SP charset) ")" ] /
-      #                   capability-data / "PARSE" /
-      #                   "PERMANENTFLAGS" SP "("
-      #                   [flag-perm *(SP flag-perm)] ")" /
-      #                   "READ-ONLY" / "READ-WRITE" / "TRYCREATE" /
-      #                   "UIDNEXT" SP nz-number / "UIDVALIDITY" SP nz-number /
-      #                   "UNSEEN" SP nz-number /
-      #                   atom [SP 1*<any TEXT-CHAR except "]">]
+      # RFC5530:
+      #   resp-text-code  =/ "UNAVAILABLE" / "AUTHENTICATIONFAILED" /
+      #                     "AUTHORIZATIONFAILED" / "EXPIRED" /
+      #                     "PRIVACYREQUIRED" / "CONTACTADMIN" / "NOPERM" /
+      #                     "INUSE" / "EXPUNGEISSUED" / "CORRUPTION" /
+      #                     "SERVERBUG" / "CLIENTBUG" / "CANNOT" /
+      #                     "LIMIT" / "OVERQUOTA" / "ALREADYEXISTS" /
+      #                     "NONEXISTENT"
+      # RFC9051:
+      #   resp-text-code   = "ALERT" /
+      #                      "BADCHARSET" [SP "(" charset *(SP charset) ")" ] /
+      #                      capability-data / "PARSE" /
+      #                      "PERMANENTFLAGS" SP "(" [flag-perm *(SP flag-perm)] ")" /
+      #                      "READ-ONLY" / "READ-WRITE" / "TRYCREATE" /
+      #                      "UIDNEXT" SP nz-number / "UIDVALIDITY" SP nz-number /
+      #                      resp-code-apnd / resp-code-copy / "UIDNOTSTICKY" /
+      #                      "UNAVAILABLE" / "AUTHENTICATIONFAILED" /
+      #                      "AUTHORIZATIONFAILED" / "EXPIRED" /
+      #                      "PRIVACYREQUIRED" / "CONTACTADMIN" / "NOPERM" /
+      #                      "INUSE" / "EXPUNGEISSUED" / "CORRUPTION" /
+      #                      "SERVERBUG" / "CLIENTBUG" / "CANNOT" /
+      #                      "LIMIT" / "OVERQUOTA" / "ALREADYEXISTS" /
+      #                      "NONEXISTENT" / "NOTSAVED" / "HASCHILDREN" /
+      #                      "CLOSED" /
+      #                      "UNKNOWN-CTE" /
+      #                      atom [SP 1*<any TEXT-CHAR except "]">]
+      #   capability-data  = "CAPABILITY" *(SP capability) SP "IMAP4rev2"
+      #                      *(SP capability)
       #
-      # +UIDPLUS+ ABNF:: https://www.rfc-editor.org/rfc/rfc4315.html#section-4
-      #   resp-text-code  =/ resp-code-apnd / resp-code-copy / "UIDNOTSTICKY"
+      # RFC4315 (UIDPLUS), RFC9051 (IMAP4rev2):
+      #   resp-code-apnd   = "APPENDUID" SP nz-number SP append-uid
+      #   resp-code-copy   = "COPYUID" SP nz-number SP uid-set SP uid-set
+      #   resp-text-code   =/ resp-code-apnd / resp-code-copy / "UIDNOTSTICKY"
+      #
+      # RFC7162 (CONDSTORE):
+      #   resp-text-code   =/ "HIGHESTMODSEQ" SP mod-sequence-value /
+      #                       "NOMODSEQ" /
+      #                       "MODIFIED" SP sequence-set
       def resp_text_code
-        token = match(T_ATOM)
-        name = token.value.upcase
-        case name
-        when /\A(?:ALERT|PARSE|READ-ONLY|READ-WRITE|TRYCREATE|NOMODSEQ)\z/n
-          result = ResponseCode.new(name, nil)
-        when /\A(?:BADCHARSET)\z/n
-          result = ResponseCode.new(name, charset_list)
-        when /\A(?:CAPABILITY)\z/ni
-          result = ResponseCode.new(name, capability__list)
-        when /\A(?:PERMANENTFLAGS)\z/n
-          match(T_SPACE)
-          result = ResponseCode.new(name, flag_list)
-        when /\A(?:UIDVALIDITY|UIDNEXT|UNSEEN)\z/n
-          match(T_SPACE)
-          result = ResponseCode.new(name, number)
-        when /\A(?:APPENDUID)\z/n
-          result = ResponseCode.new(name, resp_code_apnd__data)
-        when /\A(?:COPYUID)\z/n
-          result = ResponseCode.new(name, resp_code_copy__data)
-        else
-          token = lookahead
-          if token.symbol == T_SPACE
-            shift_token
-            result = ResponseCode.new(name, text_chars_except_rbra)
+        name = resp_text_code__name
+        data =
+          case name
+          when "CAPABILITY"         then resp_code__capability
+          when "PERMANENTFLAGS"     then SP? ? flag_perm__list : []
+          when "UIDNEXT"            then SP!; nz_number
+          when "UIDVALIDITY"        then SP!; nz_number
+          when "UNSEEN"             then SP!; nz_number            # rev1 only
+          when "APPENDUID"          then SP!; resp_code_apnd__data # rev2, UIDPLUS
+          when "COPYUID"            then SP!; resp_code_copy__data # rev2, UIDPLUS
+          when "BADCHARSET"         then SP? ? charset__list : []
+          when "ALERT", "PARSE", "READ-ONLY", "READ-WRITE", "TRYCREATE",
+            "UNAVAILABLE", "AUTHENTICATIONFAILED", "AUTHORIZATIONFAILED",
+            "EXPIRED", "PRIVACYREQUIRED", "CONTACTADMIN", "NOPERM", "INUSE",
+            "EXPUNGEISSUED", "CORRUPTION", "SERVERBUG", "CLIENTBUG", "CANNOT",
+            "LIMIT", "OVERQUOTA", "ALREADYEXISTS", "NONEXISTENT", "CLOSED",
+            "NOTSAVED", "UIDNOTSTICKY", "UNKNOWN-CTE", "HASCHILDREN"
+          when "NOMODSEQ"           # CONDSTORE
           else
-            result = ResponseCode.new(name, nil)
+            SP? and text_chars_except_rbra
           end
-        end
-        return result
+        ResponseCode.new(name, data)
       end
 
+      alias resp_text_code__name case_insensitive__atom
+
       # 1*<any TEXT-CHAR except "]">
       def text_chars_except_rbra
         match_re(CTEXT_REGEXP, '1*<any TEXT-CHAR except "]">')[0]
       end
 
-      def charset_list
-        result = []
-        if accept(T_SPACE)
-          match(T_LPAR)
-          result << charset
-          while accept(T_SPACE)
-            result << charset
-          end
-          match(T_RPAR)
-        end
-        result
+      # "(" charset *(SP charset) ")"
+      def charset__list
+        lpar; list = [charset]; while SP? do list << charset end; rpar; list
       end
 
       # already matched:  "APPENDUID"
@@ -1402,8 +1623,8 @@ module Net
       # match uid_set even if that returns a single-member array.
       #
       def resp_code_apnd__data
-        match(T_SPACE); validity = number
-        match(T_SPACE); dst_uids = uid_set # uniqueid ⊂ uid-set
+        validity = number; SP!
+        dst_uids = uid_set # uniqueid ⊂ uid-set
         UIDPlusData.new(validity, nil, dst_uids)
       end
 
@@ -1411,9 +1632,9 @@ module Net
       #
       # resp-code-copy  = "COPYUID" SP nz-number SP uid-set SP uid-set
       def resp_code_copy__data
-        match(T_SPACE); validity = number
-        match(T_SPACE); src_uids = uid_set
-        match(T_SPACE); dst_uids = uid_set
+        validity = number;  SP!
+        src_uids = uid_set; SP!
+        dst_uids = uid_set
         UIDPlusData.new(validity, src_uids, dst_uids)
       end
 
@@ -1472,36 +1693,56 @@ module Net
         return Address.new(name, route, mailbox, host)
       end
 
-      FLAG_REGEXP = /\
-(?# FLAG        )\\([^\x80-\xff(){ \x00-\x1f\x7f%"\\]+)|\
-(?# ATOM        )([^\x80-\xff(){ \x00-\x1f\x7f%*"\\]+)/n
-
+      # flag-list       = "(" [flag *(SP flag)] ")"
       def flag_list
-        if @str.index(/\(([^)]*)\)/ni, @pos)
-          @pos = $~.end(0)
-          return $1.scan(FLAG_REGEXP).collect { |flag, atom|
-            if atom
-              atom
-            else
-              flag.capitalize.intern
-            end
-          }
-        else
-          parse_error("invalid flag list")
-        end
+        match_re(Patterns::FLAG_LIST, "flag-list")[1]
+          .split(nil)
+          .map! { _1.start_with?("\\") ? _1[1..].capitalize.to_sym : _1 }
+      end
+
+      #   "(" [flag-perm *(SP flag-perm)] ")"
+      def flag_perm__list
+        match_re(Patterns::FLAG_PERM_LIST, "PERMANENTFLAGS flag-perm list")[1]
+          .split(nil)
+          .map! { _1.start_with?("\\") ? _1[1..].capitalize.to_sym : _1 }
+      end
+
+      # Not checking for max one mbx-list-sflag in the parser.
+      # >>>
+      #   mbx-list-flags  = *(mbx-list-oflag SP) mbx-list-sflag
+      #                     *(SP mbx-list-oflag) /
+      #                     mbx-list-oflag *(SP mbx-list-oflag)
+      #   mbx-list-oflag  = "\Noinferiors" / child-mbox-flag /
+      #                     "\Subscribed" / "\Remote" / flag-extension
+      #                  ; Other flags; multiple from this list are
+      #                  ; possible per LIST response, but each flag
+      #                  ; can only appear once per LIST response
+      #   mbx-list-sflag  = "\NonExistent" / "\Noselect" / "\Marked" /
+      #                     "\Unmarked"
+      #                  ; Selectability flags; only one per LIST response
+      def parens__mbx_list_flags
+        match_re(Patterns::MBX_LIST_FLAGS, "mbx-list-flags")[1]
+          .split(nil).map! { _1.capitalize.to_sym }
       end
 
-
       # See https://www.rfc-editor.org/errata/rfc3501
       #
       # charset = atom / quoted
-      def charset
-        if token = accept(T_QUOTED)
-          token.value
-        else
-          atom
-        end
-      end
+      def charset; quoted? || atom end
+
+      # RFC7162:
+      # mod-sequence-value  = 1*DIGIT
+      #                        ;; Positive unsigned 63-bit integer
+      #                        ;; (mod-sequence)
+      #                        ;; (1 <= n <= 9,223,372,036,854,775,807).
+      alias mod_sequence_value nz_number64
+
+      # RFC7162:
+      # permsg-modsequence  = mod-sequence-value
+      #                        ;; Per-message mod-sequence.
+      alias permsg_modsequence mod_sequence_value
+
+      def parens__modseq; lpar; _ = permsg_modsequence; rpar; _ end
 
       # RFC-4315 (UIDPLUS) or RFC9051 (IMAP4rev2):
       #      uid-set         = (uniqueid / uid-range) *("," uid-set)
@@ -1535,10 +1776,10 @@ module Net
       #
       # This advances @pos directly so it's safe before changing @lex_state.
       def accept_spaces
-        shift_token if @token&.symbol == T_SPACE
-        if @str.index(SPACES_REGEXP, @pos)
+        return false unless SP?
+        @str.index(SPACES_REGEXP, @pos) and
           @pos = $~.end(0)
-        end
+        true
       end
 
       def next_token