net-imap 0.3.7 → 0.4.7

data/lib/net/imap/response_parser.rb

@@ -1,12 +1,16 @@
 # frozen_string_literal: true
 
 require_relative "errors"
+require_relative "response_parser/parser_utils"
 
 module Net
   class IMAP < Protocol
 
     # Parses an \IMAP server response.
     class ResponseParser
+      include ParserUtils
+      extend  ParserUtils::Generator
+
       # :call-seq: Net::IMAP::ResponseParser.new -> Net::IMAP::ResponseParser
       def initialize
         @str = nil
@@ -33,742 +37,1333 @@ module Net
 
       # :stopdoc:
 
-      EXPR_BEG          = :EXPR_BEG
-      EXPR_DATA         = :EXPR_DATA
-      EXPR_TEXT         = :EXPR_TEXT
-      EXPR_RTEXT        = :EXPR_RTEXT
-      EXPR_CTEXT        = :EXPR_CTEXT
-
-      T_SPACE   = :SPACE
-      T_NIL     = :NIL
-      T_NUMBER  = :NUMBER
-      T_ATOM    = :ATOM
-      T_QUOTED  = :QUOTED
-      T_LPAR    = :LPAR
-      T_RPAR    = :RPAR
-      T_BSLASH  = :BSLASH
-      T_STAR    = :STAR
-      T_LBRA    = :LBRA
-      T_RBRA    = :RBRA
-      T_LITERAL = :LITERAL
-      T_PLUS    = :PLUS
-      T_PERCENT = :PERCENT
-      T_CRLF    = :CRLF
-      T_EOF     = :EOF
-      T_TEXT    = :TEXT
-
+      EXPR_BEG   = :EXPR_BEG     # the default, used in most places
+      EXPR_DATA  = :EXPR_DATA    # envelope, body(structure), namespaces
+
+      T_SPACE    = :SPACE        # atom special
+      T_ATOM     = :ATOM         # atom (subset of astring chars)
+      T_NIL      = :NIL          # subset of atom and label
+      T_NUMBER   = :NUMBER       # subset of atom
+      T_LBRA     = :LBRA         # subset of atom
+      T_PLUS     = :PLUS         # subset of atom; tag special
+      T_RBRA     = :RBRA         # atom special; resp_special; valid astring char
+      T_QUOTED   = :QUOTED       # starts/end with atom special
+      T_BSLASH   = :BSLASH       # atom special; quoted special
+      T_LPAR     = :LPAR         # atom special; paren list delimiter
+      T_RPAR     = :RPAR         # atom special; paren list delimiter
+      T_STAR     = :STAR         # atom special; list wildcard
+      T_PERCENT  = :PERCENT      # atom special; list wildcard
+      T_LITERAL  = :LITERAL      # starts with atom special
+      T_LITERAL8 = :LITERAL8     # starts with atom char "~"
+      T_CRLF     = :CRLF         # atom special; text special; quoted special
+      T_TEXT     = :TEXT         # any char except CRLF
+      T_EOF      = :EOF          # end of response string
+
+      module ResponseConditions
+        OK      = "OK"
+        NO      = "NO"
+        BAD     = "BAD"
+        BYE     = "BYE"
+        PREAUTH = "PREAUTH"
+
+        RESP_COND_STATES      = [OK, NO, BAD              ].freeze
+        RESP_DATA_CONDS       = [OK, NO, BAD, BYE,        ].freeze
+        AUTH_CONDS            = [OK,               PREAUTH].freeze
+        GREETING_CONDS        = [OK,          BYE, PREAUTH].freeze
+        RESP_CONDS            = [OK, NO, BAD, BYE, PREAUTH].freeze
+      end
+      include ResponseConditions
+
+      module Patterns
+
+        module CharClassSubtraction
+          refine Regexp do
+            def -(rhs); /[#{source}&&[^#{rhs.source}]]/n.freeze end
+          end
+        end
+        using CharClassSubtraction
+
+        # From RFC5234, "Augmented BNF for Syntax Specifications: ABNF"
+        # >>>
+        #   ALPHA   =  %x41-5A / %x61-7A   ; A-Z / a-z
+        #   CHAR    = %x01-7F
+        #   CRLF    =  CR LF
+        #                   ; Internet standard newline
+        #   CTL     = %x00-1F / %x7F
+        #                ; controls
+        #   DIGIT   =  %x30-39
+        #                   ; 0-9
+        #   DQUOTE  =  %x22
+        #                   ; " (Double Quote)
+        #   HEXDIG  =  DIGIT / "A" / "B" / "C" / "D" / "E" / "F"
+        #   OCTET   = %x00-FF
+        #   SP      =  %x20
+        module RFC5234
+          ALPHA     = /[A-Za-z]/n
+          CHAR      = /[\x01-\x7f]/n
+          CRLF      = /\r\n/n
+          CTL       = /[\x00-\x1F\x7F]/n
+          DIGIT     = /\d/n
+          DQUOTE    = /"/n
+          HEXDIG    = /\h/
+          OCTET     = /[\x00-\xFF]/n # not using /./m for embedding purposes
+          SP        = / /n
+        end
+
+        # UTF-8, a transformation format of ISO 10646
+        # >>>
+        #   UTF8-1      = %x00-7F
+        #   UTF8-tail   = %x80-BF
+        #   UTF8-2      = %xC2-DF UTF8-tail
+        #   UTF8-3      = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /
+        #                 %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )
+        #   UTF8-4      = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /
+        #                 %xF4 %x80-8F 2( UTF8-tail )
+        #   UTF8-char   = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4
+        #   UTF8-octets = *( UTF8-char )
+        #
+        # n.b. String * Integer is used for repetition, rather than /x{3}/,
+        # because ruby 3.2's linear-time cache-based optimization doesn't work
+        # with "bounded or fixed times repetition nesting in another repetition
+        # (e.g. /(a{2,3})*/). It is an implementation issue entirely, but we
+        # believe it is hard to support this case correctly."
+        # See https://bugs.ruby-lang.org/issues/19104
+        module RFC3629
+          UTF8_1      = /[\x00-\x7f]/n # aka ASCII 7bit
+          UTF8_TAIL   = /[\x80-\xBF]/n
+          UTF8_2      = /[\xC2-\xDF]#{UTF8_TAIL}/n
+          UTF8_3      = Regexp.union(/\xE0[\xA0-\xBF]#{UTF8_TAIL}/n,
+                                     /\xED[\x80-\x9F]#{UTF8_TAIL}/n,
+                                     /[\xE1-\xEC]#{    UTF8_TAIL.source * 2}/n,
+                                     /[\xEE-\xEF]#{    UTF8_TAIL.source * 2}/n)
+          UTF8_4      = Regexp.union(/[\xF1-\xF3]#{    UTF8_TAIL.source * 3}/n,
+                                     /\xF0[\x90-\xBF]#{UTF8_TAIL.source * 2}/n,
+                                     /\xF4[\x80-\x8F]#{UTF8_TAIL.source * 2}/n)
+          UTF8_CHAR   = Regexp.union(UTF8_1, UTF8_2, UTF8_3, UTF8_4)
+          UTF8_OCTETS = /#{UTF8_CHAR}*/n
+        end
+
+        include RFC5234
+        include RFC3629
+
+        # CHAR8           = %x01-ff
+        #                     ; any OCTET except NUL, %x00
+        CHAR8             = /[\x01-\xff]/n
+
+        # list-wildcards  = "%" / "*"
+        LIST_WILDCARDS    = /[%*]/n
+        # quoted-specials = DQUOTE / "\"
+        QUOTED_SPECIALS   = /["\\]/n
+        # resp-specials   = "]"
+        RESP_SPECIALS     = /[\]]/n
+
+        # atomish         = 1*<any ATOM-CHAR except "[">
+        #                 ; We use "atomish" for msg-att and section, in order
+        #                 ; to simplify "BODY[HEADER.FIELDS (foo bar)]".
+        #
+        # atom-specials   = "(" / ")" / "{" / SP / CTL / list-wildcards /
+        #                   quoted-specials / resp-specials
+        # ATOM-CHAR       = <any CHAR except atom-specials>
+        # atom            = 1*ATOM-CHAR
+        # ASTRING-CHAR    = ATOM-CHAR / resp-specials
+        # tag             = 1*<any ASTRING-CHAR except "+">
+
+        ATOM_SPECIALS     = /[(){ \x00-\x1f\x7f%*"\\\]]/n
+        ASTRING_SPECIALS  = /[(){ \x00-\x1f\x7f%*"\\]/n
+
+        ASTRING_CHAR      = CHAR - ASTRING_SPECIALS
+        ATOM_CHAR         = CHAR - ATOM_SPECIALS
+
+        ATOM              = /#{ATOM_CHAR}+/n
+        ASTRING_CHARS     = /#{ASTRING_CHAR}+/n
+        ATOMISH           = /#{ATOM_CHAR    - /[\[]/ }+/
+        TAG               = /#{ASTRING_CHAR - /[+]/  }+/
+
+        # TEXT-CHAR       = <any CHAR except CR and LF>
+        TEXT_CHAR         = CHAR - /[\r\n]/
+
+        # resp-text-code  = ... / atom [SP 1*<any TEXT-CHAR except "]">]
+        CODE_TEXT_CHAR    = TEXT_CHAR - RESP_SPECIALS
+        CODE_TEXT         = /#{CODE_TEXT_CHAR}+/n
+
+        # flag            = "\Answered" / "\Flagged" / "\Deleted" /
+        #                   "\Seen" / "\Draft" / flag-keyword / flag-extension
+        #                     ; Does not include "\Recent"
+        # flag-extension  = "\" atom
+        #                     ; Future expansion.  Client implementations
+        #                     ; MUST accept flag-extension flags.  Server
+        #                     ; implementations MUST NOT generate
+        #                     ; flag-extension flags except as defined by
+        #                     ; a future Standard or Standards Track
+        #                     ; revisions of this specification.
+        # flag-keyword    = "$MDNSent" / "$Forwarded" / "$Junk" /
+        #                   "$NotJunk" / "$Phishing" / atom
+        #
+        # flag-perm       = flag / "\*"
+        #
+        # Not checking for max one mbx-list-sflag in the parser.
+        # >>>
+        # mbx-list-oflag  = "\Noinferiors" / child-mbox-flag /
+        #                   "\Subscribed" / "\Remote" / flag-extension
+        #                    ; Other flags; multiple from this list are
+        #                    ; possible per LIST response, but each flag
+        #                    ; can only appear once per LIST response
+        # mbx-list-sflag  = "\NonExistent" / "\Noselect" / "\Marked" /
+        #                   "\Unmarked"
+        #                    ; Selectability flags; only one per LIST response
+        # child-mbox-flag =  "\HasChildren" / "\HasNoChildren"
+        #                    ; attributes for the CHILDREN return option, at most
+        #                    ; one possible per LIST response
+        FLAG              = /\\?#{ATOM}/n
+        FLAG_EXTENSION    = /\\#{ATOM}/n
+        FLAG_KEYWORD      = ATOM
+        FLAG_PERM         = Regexp.union(FLAG, "\\*")
+        MBX_FLAG          = FLAG_EXTENSION
+
+        # flag-list       = "(" [flag *(SP flag)] ")"
+        # resp-text-code =/ "PERMANENTFLAGS" SP
+        #                   "(" [flag-perm *(SP flag-perm)] ")"
+        # mbx-list-flags  = *(mbx-list-oflag SP) mbx-list-sflag
+        #                   *(SP mbx-list-oflag) /
+        #                   mbx-list-oflag *(SP mbx-list-oflag)
+        # (Not checking for max one mbx-list-sflag in the parser.)
+        FLAG_LIST         = /\G\((#{FLAG     }(?:#{SP}#{FLAG     })*|)\)/ni
+        FLAG_PERM_LIST    = /\G\((#{FLAG_PERM}(?:#{SP}#{FLAG_PERM})*|)\)/ni
+        MBX_LIST_FLAGS    = /\G  (#{MBX_FLAG }(?:#{SP}#{MBX_FLAG })*)   /nix
+
+        # RFC3501:
+        #   QUOTED-CHAR   = <any TEXT-CHAR except quoted-specials> /
+        #                   "\" quoted-specials
+        # RFC9051:
+        #   QUOTED-CHAR   = <any TEXT-CHAR except quoted-specials> /
+        #                   "\" quoted-specials / UTF8-2 / UTF8-3 / UTF8-4
+        # RFC3501 & RFC9051:
+        #   quoted          = DQUOTE *QUOTED-CHAR DQUOTE
+        QUOTED_CHAR_safe  = TEXT_CHAR - QUOTED_SPECIALS
+        QUOTED_CHAR_esc   = /\\#{QUOTED_SPECIALS}/n
+        QUOTED_CHAR_rev1  = Regexp.union(QUOTED_CHAR_safe, QUOTED_CHAR_esc)
+        QUOTED_CHAR_rev2  = Regexp.union(QUOTED_CHAR_rev1,
+                                         UTF8_2, UTF8_3, UTF8_4)
+        QUOTED_rev1       = /"(#{QUOTED_CHAR_rev1}*)"/n
+        QUOTED_rev2       = /"(#{QUOTED_CHAR_rev2}*)"/n
+
+        # RFC3501:
+        #   text          = 1*TEXT-CHAR
+        # RFC9051:
+        #   text          = 1*(TEXT-CHAR / UTF8-2 / UTF8-3 / UTF8-4)
+        #                     ; Non-ASCII text can only be returned
+        #                     ; after ENABLE IMAP4rev2 command
+        TEXT_rev1         = /#{TEXT_CHAR}+/
+        TEXT_rev2         = /#{Regexp.union TEXT_CHAR, UTF8_2, UTF8_3, UTF8_4}+/
+
+        # tagged-label-fchar = ALPHA / "-" / "_" / "."
+        TAGGED_LABEL_FCHAR   = /[a-zA-Z\-_.]/n
+        # tagged-label-char  = tagged-label-fchar / DIGIT / ":"
+        TAGGED_LABEL_CHAR    = /[a-zA-Z\-_.0-9:]*/n
+        # tagged-ext-label   = tagged-label-fchar *tagged-label-char
+        #                      ; Is a valid RFC 3501 "atom".
+        TAGGED_EXT_LABEL     = /#{TAGGED_LABEL_FCHAR}#{TAGGED_LABEL_CHAR}*/n
+
+        # nz-number       = digit-nz *DIGIT
+        #                     ; Non-zero unsigned 32-bit integer
+        #                     ; (0 < n < 4,294,967,296)
+        NZ_NUMBER         = /[1-9]\d*/n
+
+        # seq-number      = nz-number / "*"
+        #                     ; message sequence number (COPY, FETCH, STORE
+        #                     ; commands) or unique identifier (UID COPY,
+        #                     ; UID FETCH, UID STORE commands).
+        #                     ; * represents the largest number in use.  In
+        #                     ; the case of message sequence numbers, it is
+        #                     ; the number of messages in a non-empty mailbox.
+        #                     ; In the case of unique identifiers, it is the
+        #                     ; unique identifier of the last message in the
+        #                     ; mailbox or, if the mailbox is empty, the
+        #                     ; mailbox's current UIDNEXT value.
+        #                     ; The server should respond with a tagged BAD
+        #                     ; response to a command that uses a message
+        #                     ; sequence number greater than the number of
+        #                     ; messages in the selected mailbox.  This
+        #                     ; includes "*" if the selected mailbox is empty.
+        SEQ_NUMBER        = /#{NZ_NUMBER}|\*/n
+
+        # seq-range       = seq-number ":" seq-number
+        #                     ; two seq-number values and all values between
+        #                     ; these two regardless of order.
+        #                     ; Example: 2:4 and 4:2 are equivalent and
+        #                     ; indicate values 2, 3, and 4.
+        #                     ; Example: a unique identifier sequence range of
+        #                     ; 3291:* includes the UID of the last message in
+        #                     ; the mailbox, even if that value is less than
+        #                     ; 3291.
+        SEQ_RANGE         = /#{SEQ_NUMBER}:#{SEQ_NUMBER}/n
+
+        # sequence-set    = (seq-number / seq-range) ["," sequence-set]
+        #                     ; set of seq-number values, regardless of order.
+        #                     ; Servers MAY coalesce overlaps and/or execute
+        #                     ; the sequence in any order.
+        #                     ; Example: a message sequence number set of
+        #                     ; 2,4:7,9,12:* for a mailbox with 15 messages is
+        #                     ; equivalent to 2,4,5,6,7,9,12,13,14,15
+        #                     ; Example: a message sequence number set of
+        #                     ; *:4,5:7 for a mailbox with 10 messages is
+        #                     ; equivalent to 10,9,8,7,6,5,4,5,6,7 and MAY
+        #                     ; be reordered and overlap coalesced to be
+        #                     ; 4,5,6,7,8,9,10.
+        SEQUENCE_SET_ITEM = /#{SEQ_NUMBER}|#{SEQ_RANGE}/n
+        SEQUENCE_SET      = /#{SEQUENCE_SET_ITEM}(?:,#{SEQUENCE_SET_ITEM})*/n
+        SEQUENCE_SET_STR  = /\A#{SEQUENCE_SET}\z/n
+
+        # RFC3501:
+        #   literal          = "{" number "}" CRLF *CHAR8
+        #                        ; Number represents the number of CHAR8s
+        # RFC9051:
+        #   literal          = "{" number64 ["+"] "}" CRLF *CHAR8
+        #                        ; <number64> represents the number of CHAR8s.
+        #                        ; A non-synchronizing literal is distinguished
+        #                        ; from a synchronizing literal by the presence of
+        #                        ; "+" before the closing "}".
+        #                        ; Non-synchronizing literals are not allowed when
+        #                        ; sent from server to the client.
+        LITERAL              = /\{(\d+)\}\r\n/n
+
+        # RFC3516 (BINARY):
+        #   literal8         =   "~{" number "}" CRLF *OCTET
+        #                        ; <number> represents the number of OCTETs
+        #                        ; in the response string.
+        # RFC9051:
+        #   literal8         =  "~{" number64 "}" CRLF *OCTET
+        #                        ; <number64> represents the number of OCTETs
+        #                        ; in the response string.
+        LITERAL8             = /~\{(\d+)\}\r\n/n
+
+        module_function
+
+        def unescape_quoted!(quoted)
+          quoted
+            &.gsub!(/\\(#{QUOTED_SPECIALS})/n, "\\1")
+            &.force_encoding("UTF-8")
+        end
+
+        def unescape_quoted(quoted)
+          quoted
+            &.gsub(/\\(#{QUOTED_SPECIALS})/n, "\\1")
+            &.force_encoding("UTF-8")
+        end
+
+      end
+
+      # the default, used in most places
       BEG_REGEXP = /\G(?:\
-(?# 1:  SPACE   )( +)|\
-(?# 2:  NIL     )(NIL)(?=[\x80-\xff(){ \x00-\x1f\x7f%*"\\\[\]+])|\
-(?# 3:  NUMBER  )(\d+)(?=[\x80-\xff(){ \x00-\x1f\x7f%*"\\\[\]+])|\
-(?# 4:  ATOM    )([^\x80-\xff(){ \x00-\x1f\x7f%*"\\\[\]+]+)|\
-(?# 5:  QUOTED  )"((?:[^\x00\r\n"\\]|\\["\\])*)"|\
-(?# 6:  LPAR    )(\()|\
-(?# 7:  RPAR    )(\))|\
-(?# 8:  BSLASH  )(\\)|\
-(?# 9:  STAR    )(\*)|\
-(?# 10: LBRA    )(\[)|\
-(?# 11: RBRA    )(\])|\
-(?# 12: LITERAL )\{(\d+)\}\r\n|\
-(?# 13: PLUS    )(\+)|\
-(?# 14: PERCENT )(%)|\
-(?# 15: CRLF    )(\r\n)|\
-(?# 16: EOF     )(\z))/ni
-
+(?# 1:  SPACE   )( )|\
+(?# 2:  LITERAL8)#{Patterns::LITERAL8}|\
+(?# 3:  ATOM prefixed with a compatible subtype)\
+((?:\
+(?# 4:  NIL     )(NIL)|\
+(?# 5:  NUMBER  )(\d+)|\
+(?# 6:  PLUS    )(\+))\
+(?# 7:  ATOM remaining after prefix )(#{Patterns::ATOMISH})?\
+(?# This enables greedy alternation without lookahead, in linear time.)\
+)|\
+(?# Also need to check for ATOM without a subtype prefix.)\
+(?# 8:  ATOM    )(#{Patterns::ATOMISH})|\
+(?# 9:  QUOTED  )#{Patterns::QUOTED_rev2}|\
+(?# 10: LPAR    )(\()|\
+(?# 11: RPAR    )(\))|\
+(?# 12: BSLASH  )(\\)|\
+(?# 13: STAR    )(\*)|\
+(?# 14: LBRA    )(\[)|\
+(?# 15: RBRA    )(\])|\
+(?# 16: LITERAL )#{Patterns::LITERAL}|\
+(?# 17: PERCENT )(%)|\
+(?# 18: CRLF    )(\r\n)|\
+(?# 19: EOF     )(\z))/ni
+
+      # envelope, body(structure), namespaces
       DATA_REGEXP = /\G(?:\
 (?# 1:  SPACE   )( )|\
 (?# 2:  NIL     )(NIL)|\
 (?# 3:  NUMBER  )(\d+)|\
-(?# 4:  QUOTED  )"((?:[^\x00\r\n"\\]|\\["\\])*)"|\
-(?# 5:  LITERAL )\{(\d+)\}\r\n|\
+(?# 4:  QUOTED  )#{Patterns::QUOTED_rev2}|\
+(?# 5:  LITERAL )#{Patterns::LITERAL}|\
 (?# 6:  LPAR    )(\()|\
 (?# 7:  RPAR    )(\)))/ni
 
-      TEXT_REGEXP = /\G(?:\
-(?# 1:  TEXT    )([^\x00\r\n]*))/ni
-
-      RTEXT_REGEXP = /\G(?:\
-(?# 1:  LBRA    )(\[)|\
-(?# 2:  TEXT    )([^\x00\r\n]*))/ni
+      # text, after 'resp-text-code "]"'
+      TEXT_REGEXP = /\G(#{Patterns::TEXT_rev2})/n
 
-      CTEXT_REGEXP = /\G(?:\
-(?# 1:  TEXT    )([^\x00\r\n\]]*))/ni
+      # resp-text-code, after 'atom SP'
+      CTEXT_REGEXP = /\G(#{Patterns::CODE_TEXT})/n
 
       Token = Struct.new(:symbol, :value)
 
-      def response
-        token = lookahead
-        case token.symbol
-        when T_PLUS
-          result = continue_req
-        when T_STAR
-          result = response_untagged
-        else
-          result = response_tagged
-        end
-        while lookahead.symbol == T_SPACE
-          # Ignore trailing space for Microsoft Exchange Server
-          shift_token
-        end
-        match(T_CRLF)
-        match(T_EOF)
-        return result
-      end
+      def_char_matchers :SP,   " ", :T_SPACE
+      def_char_matchers :PLUS, "+", :T_PLUS
+      def_char_matchers :STAR, "*", :T_STAR
 
-      def continue_req
-        match(T_PLUS)
-        token = lookahead
-        if token.symbol == T_SPACE
-          shift_token
-          return ContinuationRequest.new(resp_text, @str)
+      def_char_matchers :lpar, "(", :T_LPAR
+      def_char_matchers :rpar, ")", :T_RPAR
+
+      def_char_matchers :lbra, "[", :T_LBRA
+      def_char_matchers :rbra, "]", :T_RBRA
+
+      # valid number ranges are not enforced by parser
+      #   number          = 1*DIGIT
+      #                       ; Unsigned 32-bit integer
+      #                       ; (0 <= n < 4,294,967,296)
+      def_token_matchers :number, T_NUMBER, coerce: Integer
+
+      def_token_matchers :quoted, T_QUOTED
+
+      #   string          = quoted / literal
+      def_token_matchers :string,  T_QUOTED, T_LITERAL
+
+      # used by nstring8 = nstring / literal8
+      def_token_matchers :string8, T_QUOTED, T_LITERAL, T_LITERAL8
+
+      # use where string represents "LABEL" values
+      def_token_matchers :case_insensitive__string,
+                         T_QUOTED, T_LITERAL,
+                         send: :upcase
+
+      # n.b: NIL? and NIL! return the "NIL" atom string (truthy) on success.
+      # NIL? returns nil when it does *not* match
+      def_token_matchers :NIL, T_NIL
+
+      # In addition to explicitly uses of +tagged-ext-label+, use this to match
+      # keywords when the grammar has not provided any extension syntax.
+      #
+      # Do *not* use this for labels where the grammar specifies extensions
+      # can be +atom+, even if all currently defined labels would match.  For
+      # example response codes in +resp-text-code+.
+      #
+      #   tagged-ext-label    = tagged-label-fchar *tagged-label-char
+      #                         ; Is a valid RFC 3501 "atom".
+      #   tagged-label-fchar  = ALPHA / "-" / "_" / "."
+      #   tagged-label-char   = tagged-label-fchar / DIGIT / ":"
+      #
+      # TODO: add to lexer and only match tagged-ext-label
+      def_token_matchers :tagged_ext_label, T_ATOM, T_NIL, send: :upcase
+
+      def_token_matchers :CRLF, T_CRLF
+      def_token_matchers :EOF,  T_EOF
+
+      # atom            = 1*ATOM-CHAR
+      # ATOM-CHAR       = <any CHAR except atom-specials>
+      ATOM_TOKENS = [T_ATOM, T_NUMBER, T_NIL, T_LBRA, T_PLUS]
+
+      SEQUENCE_SET_TOKENS = [T_ATOM, T_NUMBER, T_STAR]
+
+      #   sequence-set    = (seq-number / seq-range) ["," sequence-set]
+      #   sequence-set    =/ seq-last-command
+      #                       ; Allow for "result of the last command"
+      #                       ; indicator.
+      #   seq-last-command   = "$"
+      #
+      # *note*: doesn't match seq-last-command
+      def sequence_set
+        str = combine_adjacent(*SEQUENCE_SET_TOKENS)
+        if Patterns::SEQUENCE_SET_STR.match?(str)
+          SequenceSet.new(str)
         else
-          return ContinuationRequest.new(ResponseText.new(nil, ""), @str)
+          parse_error("unexpected atom %p, expected sequence-set", str)
         end
       end
 
-      def response_untagged
-        match(T_STAR)
-        match(T_SPACE)
-        token = lookahead
-        if token.symbol == T_NUMBER
-          return numeric_response
-        elsif token.symbol == T_ATOM
-          case token.value
-          when /\A(?:OK|NO|BAD|BYE|PREAUTH)\z/ni
-            return response_cond
-          when /\A(?:FLAGS)\z/ni
-            return flags_response
-          when /\A(?:ID)\z/ni
-            return id_response
-          when /\A(?:LIST|LSUB|XLIST)\z/ni
-            return list_response
-          when /\A(?:NAMESPACE)\z/ni
-            return namespace_response
-          when /\A(?:QUOTA)\z/ni
-            return getquota_response
-          when /\A(?:QUOTAROOT)\z/ni
-            return getquotaroot_response
-          when /\A(?:ACL)\z/ni
-            return getacl_response
-          when /\A(?:SEARCH|SORT)\z/ni
-            return search_response
-          when /\A(?:THREAD)\z/ni
-            return thread_response
-          when /\A(?:STATUS)\z/ni
-            return status_response
-          when /\A(?:CAPABILITY)\z/ni
-            return capability_response
-          when /\A(?:NOOP)\z/ni
-            return ignored_response
-          else
-            return text_response
-          end
-        else
-          parse_error("unexpected token %s", token.symbol)
-        end
+      # ASTRING-CHAR    = ATOM-CHAR / resp-specials
+      # resp-specials   = "]"
+      ASTRING_CHARS_TOKENS = [*ATOM_TOKENS, T_RBRA].freeze
+
+      ASTRING_TOKENS = [T_QUOTED, *ASTRING_CHARS_TOKENS, T_LITERAL].freeze
+
+      # tag             = 1*<any ASTRING-CHAR except "+">
+      TAG_TOKENS = (ASTRING_CHARS_TOKENS - [T_PLUS]).freeze
+
+      # TODO: handle atom, astring_chars, and tag entirely inside the lexer
+      def atom;          combine_adjacent(*ATOM_TOKENS)          end
+      def astring_chars; combine_adjacent(*ASTRING_CHARS_TOKENS) end
+      def tag;           combine_adjacent(*TAG_TOKENS)           end
+
+      # the #accept version of #atom
+      def atom?; -combine_adjacent(*ATOM_TOKENS) if lookahead?(*ATOM_TOKENS) end
+
+      # Returns <tt>atom.upcase</tt>
+      def case_insensitive__atom; -combine_adjacent(*ATOM_TOKENS).upcase end
+
+      # Returns <tt>atom?&.upcase</tt>
+      def case_insensitive__atom?
+        -combine_adjacent(*ATOM_TOKENS).upcase if lookahead?(*ATOM_TOKENS)
       end
 
-      def response_tagged
-        tag = astring_chars
-        match(T_SPACE)
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        return TaggedResponse.new(tag, name, resp_text, @str)
+      #   astring         = 1*ASTRING-CHAR / string
+      def astring
+        lookahead?(*ASTRING_CHARS_TOKENS) ? astring_chars : string
       end
 
-      def response_cond
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        return UntaggedResponse.new(name, resp_text, @str)
+      def astring?
+        lookahead?(*ASTRING_CHARS_TOKENS) ? astring_chars : string?
       end
 
-      def numeric_response
-        n = number
-        match(T_SPACE)
-        token = match(T_ATOM)
-        name = token.value.upcase
-        case name
-        when "EXISTS", "RECENT", "EXPUNGE"
-          return UntaggedResponse.new(name, n, @str)
-        when "FETCH"
-          shift_token
-          match(T_SPACE)
-          data = FetchData.new(n, msg_att(n))
-          return UntaggedResponse.new(name, data, @str)
-        end
+      # Use #label or #label_in to assert specific known labels
+      # (+tagged-ext-label+ only, not +atom+).
+      def label(word)
+        (val = tagged_ext_label) == word and return val
+        parse_error("unexpected atom %p, expected %p instead", val, word)
       end
 
-      def msg_att(n)
-        match(T_LPAR)
-        attr = {}
-        while true
-          token = lookahead
-          case token.symbol
-          when T_RPAR
-            shift_token
-            break
-          when T_SPACE
-            shift_token
-            next
-          end
-          case token.value
-          when /\A(?:ENVELOPE)\z/ni
-            name, val = envelope_data
-          when /\A(?:FLAGS)\z/ni
-            name, val = flags_data
-          when /\A(?:INTERNALDATE)\z/ni
-            name, val = internaldate_data
-          when /\A(?:RFC822(?:\.HEADER|\.TEXT)?)\z/ni
-            name, val = rfc822_text
-          when /\A(?:RFC822\.SIZE)\z/ni
-            name, val = rfc822_size
-          when /\A(?:BODY(?:STRUCTURE)?)\z/ni
-            name, val = body_data
-          when /\A(?:UID)\z/ni
-            name, val = uid_data
-          when /\A(?:MODSEQ)\z/ni
-            name, val = modseq_data
-          else
-            parse_error("unknown attribute `%s' for {%d}", token.value, n)
-          end
-          attr[name] = val
-        end
-        return attr
+      # Use #label or #label_in to assert specific known labels
+      # (+tagged-ext-label+ only, not +atom+).
+      def label_in(*labels)
+        lbl = tagged_ext_label and labels.include?(lbl) and return lbl
+        parse_error("unexpected atom %p, expected one of %s instead",
+                    lbl, labels.join(" or "))
       end
 
-      def envelope_data
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        return name, envelope
+      # expects "OK" or "PREAUTH" and raises InvalidResponseError on failure
+      def resp_cond_auth__name
+        lbl = tagged_ext_label and AUTH_CONDS.include? lbl and return lbl
+        raise InvalidResponseError, "bad response type %p, expected %s" % [
+          lbl, AUTH_CONDS.join(" or ")
+        ]
       end
 
-      def envelope
-        @lex_state = EXPR_DATA
-        token = lookahead
-        if token.symbol == T_NIL
-          shift_token
-          result = nil
-        else
-          match(T_LPAR)
-          date = nstring
-          match(T_SPACE)
-          subject = nstring
-          match(T_SPACE)
-          from = address_list
-          match(T_SPACE)
-          sender = address_list
-          match(T_SPACE)
-          reply_to = address_list
-          match(T_SPACE)
-          to = address_list
-          match(T_SPACE)
-          cc = address_list
-          match(T_SPACE)
-          bcc = address_list
-          match(T_SPACE)
-          in_reply_to = nstring
-          match(T_SPACE)
-          message_id = nstring
-          match(T_RPAR)
-          result = Envelope.new(date, subject, from, sender, reply_to,
-                                to, cc, bcc, in_reply_to, message_id)
-        end
-        @lex_state = EXPR_BEG
-        return result
+      # expects "OK" or "NO" or "BAD" and raises InvalidResponseError on failure
+      def resp_cond_state__name
+        lbl = tagged_ext_label and RESP_COND_STATES.include? lbl and return lbl
+        raise InvalidResponseError, "bad response type %p, expected %s" % [
+          lbl, RESP_COND_STATES.join(" or ")
+        ]
       end
 
-      def flags_data
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        return name, flag_list
+      #   nstring         = string / nil
+      def nstring
+        NIL? ? nil : string
       end
 
-      def internaldate_data
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        token = match(T_QUOTED)
-        return name, token.value
+      def nstring8
+        NIL? ? nil : string8
       end
 
-      def rfc822_text
-        token = match(T_ATOM)
-        name = token.value.upcase
-        token = lookahead
-        if token.symbol == T_LBRA
-          shift_token
-          match(T_RBRA)
-        end
-        match(T_SPACE)
-        return name, nstring
+      def nquoted
+        NIL? ? nil : quoted
       end
 
-      def rfc822_size
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        return name, number
+      # use where nstring represents "LABEL" values
+      def case_insensitive__nstring
+        NIL? ? nil : case_insensitive__string
       end
 
-      def body_data
-        token = match(T_ATOM)
-        name = token.value.upcase
-        token = lookahead
-        if token.symbol == T_SPACE
-          shift_token
-          return name, body
-        end
-        name.concat(section)
-        token = lookahead
-        if token.symbol == T_ATOM
-          name.concat(token.value)
-          shift_token
+      # tagged-ext-comp     = astring /
+      #                       tagged-ext-comp *(SP tagged-ext-comp) /
+      #                       "(" tagged-ext-comp ")"
+      #                       ; Extensions that follow this general
+      #                       ; syntax should use nstring instead of
+      #                       ; astring when appropriate in the context
+      #                       ; of the extension.
+      #                       ; Note that a message set or a "number"
+      #                       ; can always be represented as an "atom".
+      #                       ; A URL should be represented as
+      #                       ; a "quoted" string.
+      def tagged_ext_comp
+        vals = []
+        while true
+          vals << case lookahead!(*ASTRING_TOKENS, T_LPAR).symbol
+                  when T_LPAR   then lpar; ary = tagged_ext_comp; rpar; ary
+                  when T_NUMBER then number
+                  else               astring
+                  end
+          SP? or break
         end
-        match(T_SPACE)
-        data = nstring
-        return name, data
+        vals
       end
 
-      def body
-        @lex_state = EXPR_DATA
-        token = lookahead
-        if token.symbol == T_NIL
-          shift_token
-          result = nil
+      # tagged-ext-simple is a subset of atom
+      # TODO: recognize sequence-set in the lexer
+      #
+      # tagged-ext-simple   = sequence-set / number / number64
+      def tagged_ext_simple
+        number? || sequence_set
+      end
+
+      # tagged-ext-val      = tagged-ext-simple /
+      #                       "(" [tagged-ext-comp] ")"
+      def tagged_ext_val
+        if lpar?
+          _ = peek_rpar? ? [] : tagged_ext_comp
+          rpar
+          _
         else
-          match(T_LPAR)
-          token = lookahead
-          if token.symbol == T_LPAR
-            result = body_type_mpart
-          else
-            result = body_type_1part
+          tagged_ext_simple
+        end
+      end
+
+      # mailbox         = "INBOX" / astring
+      #                     ; INBOX is case-insensitive.  All case variants of
+      #                     ; INBOX (e.g., "iNbOx") MUST be interpreted as INBOX
+      #                     ; not as an astring.  An astring which consists of
+      #                     ; the case-insensitive sequence "I" "N" "B" "O" "X"
+      #                     ; is considered to be INBOX and not an astring.
+      #                     ;  Refer to section 5.1 for further
+      #                     ; semantic details of mailbox names.
+      alias mailbox astring
+
+      # valid number ranges are not enforced by parser
+      #   number64        = 1*DIGIT
+      #                       ; Unsigned 63-bit integer
+      #                       ; (0 <= n <= 9,223,372,036,854,775,807)
+      alias number64    number
+      alias number64?   number?
+
+      # valid number ranges are not enforced by parser
+      #   nz-number       = digit-nz *DIGIT
+      #                       ; Non-zero unsigned 32-bit integer
+      #                       ; (0 < n < 4,294,967,296)
+      alias nz_number   number
+      alias nz_number?  number?
+
+      # valid number ranges are not enforced by parser
+      #   nz-number64     = digit-nz *DIGIT
+      #                       ; Unsigned 63-bit integer
+      #                       ; (0 < n <= 9,223,372,036,854,775,807)
+      alias nz_number64 nz_number
+
+      # valid number ranges are not enforced by parser
+      #      uniqueid        = nz-number
+      #                          ; Strictly ascending
+      alias uniqueid    nz_number
+
+      # valid number ranges are not enforced by parser
+      #
+      # a 64-bit unsigned integer and is the decimal equivalent for the ID hex
+      # string used in the web interface and the Gmail API.
+      alias x_gm_id     number
+
+      # [RFC3501 & RFC9051:]
+      #   response        = *(continue-req / response-data) response-done
+      #
+      # For simplicity, response isn't interpreted as the combination of the
+      # three response types, but instead represents any individual server
+      # response.  Our simplified interpretation is defined as:
+      #   response        = continue-req | response_data | response-tagged
+      #
+      # n.b: our "response-tagged" definition parses "greeting" too.
+      def response
+        resp = case lookahead!(T_PLUS, T_STAR, *TAG_TOKENS).symbol
+               when T_PLUS then continue_req
+               when T_STAR then response_data
+               else             response_tagged
+               end
+        accept_spaces # QUIRKY: Ignore trailing space (MS Exchange Server?)
+        CRLF!
+        EOF!
+        resp
+      end
+
+      # RFC3501 & RFC9051:
+      #   continue-req    = "+" SP (resp-text / base64) CRLF
+      #
+      # n.b: base64 is valid resp-text.  And in the spirit of RFC9051 Appx E 23
+      # (and to workaround existing servers), we use the following grammar:
+      #
+      #   continue-req    = "+" (SP (resp-text)) CRLF
+      def continue_req
+        PLUS!
+        ContinuationRequest.new(SP? ? resp_text : ResponseText::EMPTY, @str)
+      end
+
+      RE_RESPONSE_TYPE = /\G(?:\d+ )?(?<type>#{Patterns::TAGGED_EXT_LABEL})/n
+
+      # [RFC3501:]
+      #   response-data    = "*" SP (resp-cond-state / resp-cond-bye /
+      #                      mailbox-data / message-data / capability-data) CRLF
+      # [RFC4466:]
+      #   response-data    = "*" SP response-payload CRLF
+      #   response-payload = resp-cond-state / resp-cond-bye /
+      #                       mailbox-data / message-data / capability-data
+      # RFC5161 (ENABLE capability):
+      #   response-data    =/ "*" SP enable-data CRLF
+      # RFC5255 (LANGUAGE capability)
+      #   response-payload =/ language-data
+      # RFC5255 (I18NLEVEL=1 and I18NLEVEL=2 capabilities)
+      #   response-payload =/ comparator-data
+      # [RFC9051:]
+      #   response-data    = "*" SP (resp-cond-state / resp-cond-bye /
+      #                      mailbox-data / message-data / capability-data /
+      #                      enable-data) CRLF
+      #
+      # [merging in greeting and response-fatal:]
+      #   greeting         =  "*" SP (resp-cond-auth / resp-cond-bye) CRLF
+      #   response-fatal   =  "*" SP resp-cond-bye CRLF
+      #   response-data    =/ "*" SP (resp-cond-auth / resp-cond-bye) CRLF
+      # [removing duplicates, this is simply]
+      #   response-payload =/ resp-cond-auth
+      #
+      # TODO: remove resp-cond-auth and handle greeting separately
+      def response_data
+        STAR!; SP!
+        m = peek_re(RE_RESPONSE_TYPE) or parse_error("unparsable response")
+        case m["type"].upcase
+        when "OK"         then resp_cond_state__untagged # RFC3501, RFC9051
+        when "FETCH"      then message_data__fetch       # RFC3501, RFC9051
+        when "EXPUNGE"    then message_data__expunge     # RFC3501, RFC9051
+        when "EXISTS"     then mailbox_data__exists      # RFC3501, RFC9051
+        when "ESEARCH"    then esearch_response          # RFC4731, RFC9051, etc
+        when "VANISHED"   then expunged_resp             # RFC7162
+        when "UIDFETCH"   then uidfetch_resp             # (draft) UIDONLY
+        when "SEARCH"     then mailbox_data__search      # RFC3501 (obsolete)
+        when "CAPABILITY" then capability_data__untagged # RFC3501, RFC9051
+        when "FLAGS"      then mailbox_data__flags       # RFC3501, RFC9051
+        when "LIST"       then mailbox_data__list        # RFC3501, RFC9051
+        when "STATUS"     then mailbox_data__status      # RFC3501, RFC9051
+        when "NAMESPACE"  then namespace_response        # RFC2342, RFC9051
+        when "ENABLED"    then enable_data               # RFC5161, RFC9051
+        when "BAD"        then resp_cond_state__untagged # RFC3501, RFC9051
+        when "NO"         then resp_cond_state__untagged # RFC3501, RFC9051
+        when "PREAUTH"    then resp_cond_auth            # RFC3501, RFC9051
+        when "BYE"        then resp_cond_bye             # RFC3501, RFC9051
+        when "RECENT"     then mailbox_data__recent      # RFC3501 (obsolete)
+        when "SORT"       then sort_data                 # RFC5256, RFC7162
+        when "THREAD"     then thread_data               # RFC5256
+        when "QUOTA"      then quota_response            # RFC2087, RFC9208
+        when "QUOTAROOT"  then quotaroot_response        # RFC2087, RFC9208
+        when "ID"         then id_response               # RFC2971
+        when "ACL"        then acl_data                  # RFC4314
+        when "LISTRIGHTS" then listrights_data           # RFC4314
+        when "MYRIGHTS"   then myrights_data             # RFC4314
+        when "METADATA"   then metadata_resp             # RFC5464
+        when "LANGUAGE"   then language_data             # RFC5255
+        when "COMPARATOR" then comparator_data           # RFC5255
+        when "CONVERTED"  then message_data__converted   # RFC5259
+        when "LSUB"       then mailbox_data__lsub        # RFC3501 (obsolete)
+        when "XLIST"      then mailbox_data__xlist       # deprecated
+        when "NOOP"       then response_data__noop
+        else                   response_data__unhandled
+        end
+      end
+
+      def response_data__unhandled(klass = UntaggedResponse)
+        num  = number?;          SP?
+        type = tagged_ext_label; SP?
+        text = remaining_unparsed
+        data =
+          if num && text then UnparsedNumericResponseData.new(num, text)
+          elsif     text then UnparsedData.new(text)
+          else                num
           end
-          match(T_RPAR)
-        end
-        @lex_state = EXPR_BEG
-        return result
+        klass.new(type, data, @str)
       end
 
-      def body_type_1part
-        token = lookahead
-        case token.value
-        when /\A(?:TEXT)\z/ni
-          return body_type_text
-        when /\A(?:MESSAGE)\z/ni
-          return body_type_msg
-        when /\A(?:ATTACHMENT)\z/ni
-          return body_type_attachment
-        when /\A(?:MIXED)\z/ni
-          return body_type_mixed
-        else
-          return body_type_basic
-        end
+      # reads all the way up until CRLF
+      def remaining_unparsed
+        str = @str[@pos...-2] and @pos += str.bytesize
+        str&.empty? ? nil : str
       end
 
-      def body_type_basic
-        mtype, msubtype = media_type
-        token = lookahead
-        if token.symbol == T_RPAR
-          return BodyTypeBasic.new(mtype, msubtype)
-        end
-        match(T_SPACE)
-        param, content_id, desc, enc, size = body_fields
-        md5, disposition, language, extension = body_ext_1part
-        return BodyTypeBasic.new(mtype, msubtype,
-                                 param, content_id,
-                                 desc, enc, size,
-                                 md5, disposition, language, extension)
+      def response_data__ignored; response_data__unhandled(IgnoredResponse) end
+      alias response_data__noop     response_data__ignored
+
+      alias esearch_response        response_data__unhandled
+      alias expunged_resp           response_data__unhandled
+      alias uidfetch_resp           response_data__unhandled
+      alias listrights_data         response_data__unhandled
+      alias myrights_data           response_data__unhandled
+      alias metadata_resp           response_data__unhandled
+      alias language_data           response_data__unhandled
+      alias comparator_data         response_data__unhandled
+      alias message_data__converted response_data__unhandled
+
+      # RFC3501 & RFC9051:
+      #   response-tagged = tag SP resp-cond-state CRLF
+      def response_tagged
+        TaggedResponse.new(tag, *(SP!; resp_cond_state), @str)
       end
 
-      def body_type_text
-        mtype, msubtype = media_type
-        match(T_SPACE)
-        param, content_id, desc, enc, size = body_fields
-        match(T_SPACE)
-        lines = number
-        md5, disposition, language, extension = body_ext_1part
-        return BodyTypeText.new(mtype, msubtype,
-                                param, content_id,
-                                desc, enc, size,
-                                lines,
-                                md5, disposition, language, extension)
+      # RFC3501 & RFC9051:
+      #   resp-cond-state  = ("OK" / "NO" / "BAD") SP resp-text
+      #
+      # NOTE: In the spirit of RFC9051 Appx E 23 (and to workaround existing
+      # servers), we don't require a final SP and instead parse this as:
+      #
+      #   resp-cond-state = ("OK" / "NO" / "BAD") [SP resp-text]
+      def resp_cond_state
+        [resp_cond_state__name, SP? ? resp_text : ResponseText::EMPTY]
       end
 
-      def body_type_msg
-        mtype, msubtype = media_type
-        match(T_SPACE)
-        param, content_id, desc, enc, size = body_fields
+      def resp_cond_state__untagged
+        UntaggedResponse.new(*resp_cond_state, @str)
+      end
 
-        token = lookahead
-        if token.symbol == T_RPAR
-          # If this is not message/rfc822, we shouldn't apply the RFC822
-          # spec to it.  We should handle anything other than
-          # message/rfc822 using multipart extension data [rfc3501] (i.e.
-          # the data itself won't be returned, we would have to retrieve it
-          # with BODYSTRUCTURE instead of with BODY
-
-          # Also, sometimes a message/rfc822 is included as a large
-          # attachment instead of having all of the other details
-          # (e.g. attaching a .eml file to an email)
-          if msubtype == "RFC822"
-            return BodyTypeMessage.new(mtype, msubtype, param, content_id,
-                                       desc, enc, size, nil, nil, nil, nil,
-                                       nil, nil, nil)
-          else
-            return BodyTypeExtension.new(mtype, msubtype,
-                                         param, content_id,
-                                         desc, enc, size)
-          end
-        end
+      #   resp-cond-auth   = ("OK" / "PREAUTH") SP resp-text
+      #
+      # NOTE: In the spirit of RFC9051 Appx E 23 (and to workaround existing
+      # servers), we don't require a final SP and instead parse this as:
+      #
+      #   resp-cond-auth   = ("OK" / "PREAUTH") [SP resp-text]
+      def resp_cond_auth
+        UntaggedResponse.new(resp_cond_auth__name,
+                             SP? ? resp_text : ResponseText::EMPTY,
+                             @str)
+      end
 
-        match(T_SPACE)
-        env = envelope
-        match(T_SPACE)
-        b = body
-        match(T_SPACE)
-        lines = number
-        md5, disposition, language, extension = body_ext_1part
-        return BodyTypeMessage.new(mtype, msubtype,
-                                   param, content_id,
-                                   desc, enc, size,
-                                   env, b, lines,
-                                   md5, disposition, language, extension)
+      #   resp-cond-bye    = "BYE" SP resp-text
+      #
+      # NOTE: In the spirit of RFC9051 Appx E 23 (and to workaround existing
+      # servers), we don't require a final SP and instead parse this as:
+      #
+      #   resp-cond-bye    = "BYE" [SP resp-text]
+      def resp_cond_bye
+        UntaggedResponse.new(label(BYE),
+                             SP? ? resp_text : ResponseText::EMPTY,
+                             @str)
       end
 
-      def body_type_attachment
-        mtype = case_insensitive_string
-        match(T_SPACE)
-        param = body_fld_param
-        return BodyTypeAttachment.new(mtype, nil, param)
+      #   message-data    = nz-number SP ("EXPUNGE" / ("FETCH" SP msg-att))
+      def message_data__fetch
+        seq  = nz_number;     SP!
+        name = label "FETCH"; SP!
+        data = FetchData.new(seq, msg_att(seq))
+        UntaggedResponse.new(name, data, @str)
       end
 
-      def body_type_mixed
-        mtype = "MULTIPART"
-        msubtype = case_insensitive_string
-        param, disposition, language, extension = body_ext_mpart
-        return BodyTypeBasic.new(mtype, msubtype, param, nil, nil, nil, nil, nil, disposition, language, extension)
+      def response_data__simple_numeric
+        data = nz_number; SP!
+        name = tagged_ext_label
+        UntaggedResponse.new(name, data, @str)
       end
 
-      def body_type_mpart
-        parts = []
+      alias message_data__expunge response_data__simple_numeric
+      alias mailbox_data__exists  response_data__simple_numeric
+      alias mailbox_data__recent  response_data__simple_numeric
+
+      # RFC3501 & RFC9051:
+      #   msg-att         = "(" (msg-att-dynamic / msg-att-static)
+      #                      *(SP (msg-att-dynamic / msg-att-static)) ")"
+      #
+      #   msg-att-dynamic = "FLAGS" SP "(" [flag-fetch *(SP flag-fetch)] ")"
+      # RFC5257 (ANNOTATE extension):
+      #   msg-att-dynamic =/ "ANNOTATION" SP
+      #                        ( "(" entry-att *(SP entry-att) ")" /
+      #                          "(" entry *(SP entry) ")" )
+      # RFC7162 (CONDSTORE extension):
+      #   msg-att-dynamic =/ fetch-mod-resp
+      #   fetch-mod-resp  = "MODSEQ" SP "(" permsg-modsequence ")"
+      # RFC8970 (PREVIEW extension):
+      #   msg-att-dynamic =/ "PREVIEW" SP nstring
+      #
+      # RFC3501:
+      #   msg-att-static  = "ENVELOPE" SP envelope /
+      #                     "INTERNALDATE" SP date-time /
+      #                     "RFC822" [".HEADER" / ".TEXT"] SP nstring /
+      #                     "RFC822.SIZE" SP number /
+      #                     "BODY" ["STRUCTURE"] SP body /
+      #                     "BODY" section ["<" number ">"] SP nstring /
+      #                     "UID" SP uniqueid
+      # RFC3516 (BINARY extension):
+      #   msg-att-static  =/ "BINARY" section-binary SP (nstring / literal8)
+      #                    / "BINARY.SIZE" section-binary SP number
+      # RFC8514 (SAVEDATE extension):
+      #   msg-att-static  =/ "SAVEDATE" SP (date-time / nil)
+      # RFC8474 (OBJECTID extension):
+      #   msg-att-static =/ fetch-emailid-resp / fetch-threadid-resp
+      #   fetch-emailid-resp  = "EMAILID" SP "(" objectid ")"
+      #   fetch-threadid-resp = "THREADID" SP ( "(" objectid ")" / nil )
+      # RFC9051:
+      #   msg-att-static  = "ENVELOPE" SP envelope /
+      #                     "INTERNALDATE" SP date-time /
+      #                     "RFC822.SIZE" SP number64 /
+      #                     "BODY" ["STRUCTURE"] SP body /
+      #                     "BODY" section ["<" number ">"] SP nstring /
+      #                     "BINARY" section-binary SP (nstring / literal8) /
+      #                     "BINARY.SIZE" section-binary SP number /
+      #                     "UID" SP uniqueid
+      #
+      # Re https://www.rfc-editor.org/errata/eid7246, I'm adding "offset" to the
+      # official "BINARY" ABNF, like so:
+      #
+      #   msg-att-static   =/ "BINARY" section-binary ["<" number ">"] SP
+      #                       (nstring / literal8)
+      def msg_att(n)
+        lpar
+        attr = {}
         while true
-          token = lookahead
-          if token.symbol == T_SPACE
-            shift_token
-            break
-          end
-          parts.push(body)
+          name = msg_att__label; SP!
+          val =
+            case name
+            when "UID"                  then uniqueid
+            when "FLAGS"                then flag_list
+            when "BODY"                 then body
+            when /\ABODY\[/ni           then nstring
+            when "BODYSTRUCTURE"        then body
+            when "ENVELOPE"             then envelope
+            when "INTERNALDATE"         then date_time
+            when "RFC822.SIZE"          then number64
+            when /\ABINARY\[/ni         then nstring8           # BINARY, IMAP4rev2
+            when /\ABINARY\.SIZE\[/ni   then number             # BINARY, IMAP4rev2
+            when "RFC822"               then nstring            # not in rev2
+            when "RFC822.HEADER"        then nstring            # not in rev2
+            when "RFC822.TEXT"          then nstring            # not in rev2
+            when "MODSEQ"               then parens__modseq     # CONDSTORE
+            when "EMAILID"              then parens__objectid   # OBJECTID
+            when "THREADID"             then nparens__objectid  # OBJECTID
+            when "X-GM-MSGID"           then x_gm_id            # GMail
+            when "X-GM-THRID"           then x_gm_id            # GMail
+            when "X-GM-LABELS"          then x_gm_labels        # GMail
+            else parse_error("unknown attribute `%s' for {%d}", name, n)
+            end
+          attr[name] = val
+          break unless SP?
+          break if lookahead_rpar?
         end
-        mtype = "MULTIPART"
-        msubtype = case_insensitive_string
-        param, disposition, language, extension = body_ext_mpart
-        return BodyTypeMultipart.new(mtype, msubtype, parts,
-                                     param, disposition, language,
-                                     extension)
+        rpar
+        attr
       end
 
-      def media_type
-        mtype = case_insensitive_string
-        token = lookahead
-        if token.symbol != T_SPACE
-          return mtype, nil
+      # appends "[section]" and "<partial>" to the base label
+      def msg_att__label
+        case (name = tagged_ext_label)
+        when /\A(?:RFC822(?:\.HEADER|\.TEXT)?)\z/ni
+          # ignoring "[]" fixes https://bugs.ruby-lang.org/issues/5620
+          lbra? and rbra
+        when "BODY"
+          peek_lbra? and name << section and
+            peek_str?("<") and name << gt__number__lt # partial
+        when "BINARY", "BINARY.SIZE"
+          name << section_binary
+          # see https://www.rfc-editor.org/errata/eid7246 and the note above
+          peek_str?("<") and name << gt__number__lt # partial
         end
-        match(T_SPACE)
-        msubtype = case_insensitive_string
-        return mtype, msubtype
+        name
       end
 
-      def body_fields
-        param = body_fld_param
-        match(T_SPACE)
-        content_id = nstring
-        match(T_SPACE)
-        desc = nstring
-        match(T_SPACE)
-        enc = case_insensitive_string
-        match(T_SPACE)
-        size = number
-        return param, content_id, desc, enc, size
-      end
+      # this represents the partial size for BODY or BINARY
+      alias gt__number__lt atom
 
-      def body_fld_param
-        token = lookahead
-        if token.symbol == T_NIL
-          shift_token
-          return nil
-        end
-        match(T_LPAR)
-        param = {}
-        while true
-          token = lookahead
-          case token.symbol
-          when T_RPAR
-            shift_token
-            break
-          when T_SPACE
-            shift_token
-          end
-          name = case_insensitive_string
-          match(T_SPACE)
-          val = string
-          param[name] = val
-        end
-        return param
+      # RFC3501 & RFC9051:
+      #   envelope        = "(" env-date SP env-subject SP env-from SP
+      #                     env-sender SP env-reply-to SP env-to SP env-cc SP
+      #                     env-bcc SP env-in-reply-to SP env-message-id ")"
+      def envelope
+        @lex_state = EXPR_DATA
+        lpar; date        = env_date
+        SP!;  subject     = env_subject
+        SP!;  from        = env_from
+        SP!;  sender      = env_sender
+        SP!;  reply_to    = env_reply_to
+        SP!;  to          = env_to
+        SP!;  cc          = env_cc
+        SP!;  bcc         = env_bcc
+        SP!;  in_reply_to = env_in_reply_to
+        SP!;  message_id  = env_message_id
+        rpar
+        Envelope.new(date, subject, from, sender, reply_to,
+                     to, cc, bcc, in_reply_to, message_id)
+      ensure
+        @lex_state = EXPR_BEG
       end
 
-      def body_ext_1part
-        token = lookahead
-        if token.symbol == T_SPACE
-          shift_token
-        else
-          return nil
-        end
-        md5 = nstring
+      #   env-date        = nstring
+      #   env-subject     = nstring
+      #   env-in-reply-to = nstring
+      #   env-message-id  = nstring
+      alias env_date        nstring
+      alias env_subject     nstring
+      alias env_in_reply_to nstring
+      alias env_message_id  nstring
+
+      #   env-from        = "(" 1*address ")" / nil
+      #   env-sender      = "(" 1*address ")" / nil
+      #   env-reply-to    = "(" 1*address ")" / nil
+      #   env-to          = "(" 1*address ")" / nil
+      #   env-cc          = "(" 1*address ")" / nil
+      #   env-bcc         = "(" 1*address ")" / nil
+      def nlist__address
+        return if NIL?
+        lpar; list = [address]; list << address until (quirky_SP?; rpar?)
+        list
+      end
+
+      alias env_from     nlist__address
+      alias env_sender   nlist__address
+      alias env_reply_to nlist__address
+      alias env_to       nlist__address
+      alias env_cc       nlist__address
+      alias env_bcc      nlist__address
+
+      # Used when servers erroneously send an extra SP.
+      #
+      # As of 2023-11-28, Outlook.com (still) sends SP
+      #   between +address+ in <tt>env-*</tt> lists.
+      alias quirky_SP? SP?
 
-        token = lookahead
-        if token.symbol == T_SPACE
-          shift_token
-        else
-          return md5
-        end
-        disposition = body_fld_dsp
+      #   date-time       = DQUOTE date-day-fixed "-" date-month "-" date-year
+      #                     SP time SP zone DQUOTE
+      alias date_time quoted
+      alias ndatetime nquoted
 
-        token = lookahead
-        if token.symbol == T_SPACE
-          shift_token
-        else
-          return md5, disposition
-        end
-        language = body_fld_lang
+      # RFC-3501 & RFC-9051:
+      #   body            = "(" (body-type-1part / body-type-mpart) ")"
+      def body
+        @lex_state = EXPR_DATA
+        lpar; result = peek_lpar? ? body_type_mpart : body_type_1part; rpar
+        result
+      ensure
+        @lex_state = EXPR_BEG
+      end
+      alias lookahead_body? lookahead_lpar?
 
-        token = lookahead
-        if token.symbol == T_SPACE
-          shift_token
-        else
-          return md5, disposition, language
-        end
+      # RFC-3501 & RFC9051:
+      #   body-type-1part = (body-type-basic / body-type-msg / body-type-text)
+      #                     [SP body-ext-1part]
+      def body_type_1part
+        # This regexp peek is a performance optimization.
+        # The lookahead fallback would work fine too.
+        m = peek_re(/\G(?:
+            (?<TEXT>     "TEXT"    \s "[^"]+"             )
+            |(?<MESSAGE> "MESSAGE" \s "(?:RFC822|GLOBAL)" )
+            |(?<BASIC>   "[^"]+"   \s "[^"]+"             )
+            |(?<MIXED>   "MIXED"                          )
+           )/nix)
+        choice = m&.named_captures&.compact&.keys&.first
+        # In practice, the following line should never be used. But the ABNF
+        # *does* allow literals, and this will handle them.
+        choice ||= lookahead_case_insensitive__string!
+        case choice
+        when "BASIC"   then body_type_basic # => BodyTypeBasic
+        when "MESSAGE" then body_type_msg   # => BodyTypeMessage | BodyTypeBasic
+        when "TEXT"    then body_type_text  # => BodyTypeText
+        when "MIXED"   then body_type_mixed # => BodyTypeMultipart (server bug)
+        else                body_type_basic # might be a bug; server's or ours?
+        end
+      end
+
+      # RFC-3501 & RFC9051:
+      #   body-type-basic = media-basic SP body-fields
+      def body_type_basic
+        type = media_basic # n.b. "basic" type isn't enforced here
+        if lookahead_rpar? then return BodyTypeBasic.new(*type) end # invalid
+        SP!;    flds = body_fields
+        SP? and exts = body_ext_1part
+        BodyTypeBasic.new(*type, *flds, *exts)
+      end
 
-        extension = body_extensions
-        return md5, disposition, language, extension
+      # RFC-3501 & RFC-9051:
+      #   body-type-text  = media-text SP body-fields SP body-fld-lines
+      def body_type_text
+        type = media_text
+        SP!;   flds  = body_fields
+        SP!;   lines = body_fld_lines
+        SP? and exts = body_ext_1part
+        BodyTypeText.new(*type, *flds, lines, *exts)
       end
 
-      def body_ext_mpart
-        token = lookahead
-        if token.symbol == T_SPACE
-          shift_token
-        else
-          return nil
-        end
-        param = body_fld_param
+      # RFC-3501 & RFC-9051:
+      #   body-type-msg   = media-message SP body-fields SP envelope
+      #                     SP body SP body-fld-lines
+      def body_type_msg
+        # n.b. "message/rfc822" type isn't enforced here
+        type = media_message
+        SP!; flds = body_fields
+
+        # Sometimes servers send body-type-basic when body-type-msg should be.
+        # E.g: when a message/rfc822 part has "Content-Disposition: attachment".
+        #
+        # * SP "("     --> SP envelope       --> continue as body-type-msg
+        # * ")"        --> no body-ext-1part --> completed body-type-basic
+        # * SP nstring --> SP body-fld-md5
+        #              --> SP body-ext-1part --> continue as body-type-basic
+        #
+        # It's probably better to return BodyTypeBasic---even for
+        # "message/rfc822"---than BodyTypeMessage with invalid fields.
+        unless peek_str?(" (")
+          SP? and exts = body_ext_1part
+          return BodyTypeBasic.new(*type, *flds, *exts)
+        end
+
+        SP!; env   = envelope
+        SP!; bdy   = body
+        SP!; lines = body_fld_lines
+        SP? and exts = body_ext_1part
+        BodyTypeMessage.new(*type, *flds, env, bdy, lines, *exts)
+      end
+
+      # This is a malformed body-type-mpart with no subparts.
+      def body_type_mixed
+        # warn "malformed body-type-mpart: multipart/mixed with no parts."
+        type = media_subtype # => "MIXED"
+        SP? and exts = body_ext_mpart
+        BodyTypeMultipart.new("MULTIPART", type, nil, *exts)
+      end
 
-        token = lookahead
-        if token.symbol == T_SPACE
-          shift_token
-        else
-          return param
-        end
-        disposition = body_fld_dsp
+      # RFC-3501 & RFC-9051:
+      #   body-type-mpart = 1*body SP media-subtype
+      #                     [SP body-ext-mpart]
+      def body_type_mpart
+        parts = [body]; parts << body until SP?; msubtype = media_subtype
+        SP? and exts = body_ext_mpart
+        BodyTypeMultipart.new("MULTIPART", msubtype, parts, *exts)
+      end
 
-        token = lookahead
-        if token.symbol == T_SPACE
-          shift_token
-        else
-          return param, disposition
-        end
-        language = body_fld_lang
+      # n.b. this handles both type and subtype
+      #
+      # RFC-3501 vs RFC-9051:
+      #   media-basic     = ((DQUOTE ("APPLICATION" / "AUDIO" / "IMAGE" /
+      #                     "MESSAGE" /
+      #                     "VIDEO") DQUOTE) / string) SP media-subtype
+      #   media-basic     = ((DQUOTE ("APPLICATION" / "AUDIO" / "IMAGE" /
+      #                     "FONT" / "MESSAGE" / "MODEL" /
+      #                     "VIDEO") DQUOTE) / string) SP media-subtype
+      #
+      #   media-message   = DQUOTE "MESSAGE" DQUOTE SP
+      #                     DQUOTE "RFC822" DQUOTE
+      #   media-message   = DQUOTE "MESSAGE" DQUOTE SP
+      #                     DQUOTE ("RFC822" / "GLOBAL") DQUOTE
+      #
+      # RFC-3501 & RFC-9051:
+      #   media-text      = DQUOTE "TEXT" DQUOTE SP media-subtype
+      #   media-subtype   = string
+      def media_type
+        mtype = case_insensitive__string
+        SP? or return mtype, nil # ??? quirky!
+        msubtype = media_subtype
+        return mtype, msubtype
+      end
 
-        token = lookahead
-        if token.symbol == T_SPACE
-          shift_token
-        else
-          return param, disposition, language
-        end
+      # TODO: check types
+      alias media_basic   media_type # */* --- catchall
+      alias media_message media_type # message/rfc822, message/global
+      alias media_text    media_type # text/*
+
+      alias media_subtype case_insensitive__string
 
-        extension = body_extensions
-        return param, disposition, language, extension
+      # RFC-3501 & RFC-9051:
+      #   body-fields     = body-fld-param SP body-fld-id SP body-fld-desc SP
+      #                     body-fld-enc SP body-fld-octets
+      def body_fields
+        fields = []
+        fields << body_fld_param; SP!
+        fields << body_fld_id;    SP!
+        fields << body_fld_desc;  SP!
+        fields << body_fld_enc;   SP!
+        fields << body_fld_octets
+        fields
       end
 
+      # RFC3501, RFC9051:
+      # body-fld-param  = "(" string SP string *(SP string SP string) ")" / nil
+      def body_fld_param
+        return if NIL?
+        param = {}
+        lpar
+        name = case_insensitive__string; SP!; param[name] = string
+        while SP?
+          name = case_insensitive__string; SP!; param[name] = string
+        end
+        rpar
+        param
+      end
+
+      # RFC2060
+      #   body_ext_1part  ::= body_fld_md5 [SPACE body_fld_dsp
+      #                       [SPACE body_fld_lang
+      #                       [SPACE 1#body_extension]]]
+      #                       ;; MUST NOT be returned on non-extensible
+      #                       ;; "BODY" fetch
+      # RFC3501 & RFC9051
+      #   body-ext-1part  = body-fld-md5 [SP body-fld-dsp [SP body-fld-lang
+      #                     [SP body-fld-loc *(SP body-extension)]]]
+      #                       ; MUST NOT be returned on non-extensible
+      #                       ; "BODY" fetch
+      def body_ext_1part
+        fields = [];          fields << body_fld_md5
+        SP? or return fields; fields << body_fld_dsp
+        SP? or return fields; fields << body_fld_lang
+        SP? or return fields; fields << body_fld_loc
+        SP? or return fields; fields << body_extensions
+        fields
+      end
+
+      # RFC-2060:
+      #   body_ext_mpart  = body_fld_param [SP body_fld_dsp SP body_fld_lang
+      #                     [SP 1#body_extension]]
+      #                       ;; MUST NOT be returned on non-extensible
+      #                       ;; "BODY" fetch
+      # RFC-3501 & RFC-9051:
+      #   body-ext-mpart  = body-fld-param [SP body-fld-dsp [SP body-fld-lang
+      #                     [SP body-fld-loc *(SP body-extension)]]]
+      #                       ; MUST NOT be returned on non-extensible
+      #                       ; "BODY" fetch
+      def body_ext_mpart
+        fields = [];          fields << body_fld_param
+        SP? or return fields; fields << body_fld_dsp
+        SP? or return fields; fields << body_fld_lang
+        SP? or return fields; fields << body_fld_loc
+        SP? or return fields; fields << body_extensions
+        fields
+      end
+
+      alias body_fld_desc   nstring
+      alias body_fld_id     nstring
+      alias body_fld_loc    nstring
+      alias body_fld_lines  number64 # number in 3501, number64 in 9051
+      alias body_fld_md5    nstring
+      alias body_fld_octets number
+
+      # RFC-3501 & RFC-9051:
+      #   body-fld-enc    = (DQUOTE ("7BIT" / "8BIT" / "BINARY" / "BASE64"/
+      #                     "QUOTED-PRINTABLE") DQUOTE) / string
+      alias body_fld_enc case_insensitive__string
+
+      #   body-fld-dsp    = "(" string SP body-fld-param ")" / nil
       def body_fld_dsp
-        token = lookahead
-        if token.symbol == T_NIL
-          shift_token
-          return nil
-        end
-        match(T_LPAR)
-        dsp_type = case_insensitive_string
-        match(T_SPACE)
-        param = body_fld_param
-        match(T_RPAR)
-        return ContentDisposition.new(dsp_type, param)
+        return if NIL?
+        lpar; dsp_type = case_insensitive__string
+        SP!;  param    = body_fld_param
+        rpar
+        ContentDisposition.new(dsp_type, param)
       end
 
+      #   body-fld-lang   = nstring / "(" string *(SP string) ")"
       def body_fld_lang
-        token = lookahead
-        if token.symbol == T_LPAR
-          shift_token
-          result = []
-          while true
-            token = lookahead
-            case token.symbol
-            when T_RPAR
-              shift_token
-              return result
-            when T_SPACE
-              shift_token
-            end
-            result.push(case_insensitive_string)
-          end
+        if lpar?
+          result = [case_insensitive__string]
+          result << case_insensitive__string while SP?
+          rpar
+          result
         else
-          lang = nstring
-          if lang
-            return lang.upcase
-          else
-            return lang
-          end
+          case_insensitive__nstring
         end
       end
 
+      #   body-extension *(SP body-extension)
       def body_extensions
         result = []
-        while true
-          token = lookahead
-          case token.symbol
-          when T_RPAR
-            return result
-          when T_SPACE
-            shift_token
-          end
-          result.push(body_extension)
-        end
+        result << body_extension; while SP? do result << body_extension end
+        result
       end
 
+      #   body-extension  = nstring / number / number64 /
+      #                      "(" body-extension *(SP body-extension) ")"
+      #                       ; Future expansion.  Client implementations
+      #                       ; MUST accept body-extension fields.  Server
+      #                       ; implementations MUST NOT generate
+      #                       ; body-extension fields except as defined by
+      #                       ; future Standard or Standards Track
+      #                       ; revisions of this specification.
       def body_extension
-        token = lookahead
-        case token.symbol
-        when T_LPAR
-          shift_token
-          result = body_extensions
-          match(T_RPAR)
-          return result
-        when T_NUMBER
-          return number
-        else
-          return nstring
+        if (uint = number64?) then uint
+        elsif lpar?           then exts = body_extensions; rpar; exts
+        else                       nstring
         end
       end
 
+      # section         = "[" [section-spec] "]"
       def section
-        str = String.new
-        token = match(T_LBRA)
-        str.concat(token.value)
-        token = match(T_ATOM, T_NUMBER, T_RBRA)
-        if token.symbol == T_RBRA
-          str.concat(token.value)
-          return str
-        end
-        str.concat(token.value)
-        token = lookahead
-        if token.symbol == T_SPACE
-          shift_token
-          str.concat(token.value)
-          token = match(T_LPAR)
-          str.concat(token.value)
-          while true
-            token = lookahead
-            case token.symbol
-            when T_RPAR
-              str.concat(token.value)
-              shift_token
-              break
-            when T_SPACE
-              shift_token
-              str.concat(token.value)
-            end
-            str.concat(format_string(astring))
-          end
-        end
-        token = match(T_RBRA)
-        str.concat(token.value)
-        return str
-      end
-
-      def format_string(str)
-        case str
-        when ""
-          return '""'
-        when /[\x80-\xff\r\n]/n
-          # literal
-          return "{" + str.bytesize.to_s + "}" + CRLF + str
-        when /[(){ \x00-\x1f\x7f%*"\\]/n
-          # quoted string
-          return '"' + str.gsub(/["\\]/n, "\\\\\\&") + '"'
-        else
-          # atom
-          return str
-        end
-      end
-
-      def uid_data
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        return name, number
-      end
-
-      def modseq_data
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        match(T_LPAR)
-        modseq = number
-        match(T_RPAR)
-        return name, modseq
-      end
-
-      def ignored_response
-        while lookahead.symbol != T_CRLF
-          shift_token
-        end
-        return IgnoredResponse.new(@str)
-      end
-
-      def text_response
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        return UntaggedResponse.new(name, text)
-      end
-
-      def flags_response
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        return UntaggedResponse.new(name, flag_list, @str)
-      end
-
-      def list_response
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        return UntaggedResponse.new(name, mailbox_list, @str)
-      end
-
+        str = +lbra
+        str << section_spec unless peek_rbra?
+        str << rbra
+      end
+
+      # section-binary  = "[" [section-part] "]"
+      def section_binary
+        str = +lbra
+        str << section_part unless peek_rbra?
+        str << rbra
+      end
+
+      # section-spec    = section-msgtext / (section-part ["." section-text])
+      # section-msgtext = "HEADER" /
+      #                   "HEADER.FIELDS" [".NOT"] SP header-list /
+      #                   "TEXT"
+      #                     ; top-level or MESSAGE/RFC822 or
+      #                     ; MESSAGE/GLOBAL part
+      # section-part    = nz-number *("." nz-number)
+      #                     ; body part reference.
+      #                     ; Allows for accessing nested body parts.
+      # section-text    = section-msgtext / "MIME"
+      #                     ; text other than actual body part (headers,
+      #                     ; etc.)
+      #
+      # n.b: we could "cheat" here and just grab all text inside the brackets,
+      # but literals would need special treatment.
+      def section_spec
+        str = "".b
+        str << atom # grabs everything up to "SP header-list" or "]"
+        str << " " << header_list if SP?
+        str
+      end
+
+      # header-list     = "(" header-fld-name *(SP header-fld-name) ")"
+      def header_list
+        str = +""
+        str << lpar << header_fld_name
+        str << " "  << header_fld_name while SP?
+        str << rpar
+      end
+
+      # section-part    = nz-number *("." nz-number)
+      #                     ; body part reference.
+      #                     ; Allows for accessing nested body parts.
+      alias section_part atom
+
+      # RFC3501 & RFC9051:
+      #   header-fld-name = astring
+      #
+      # NOTE: Previously, Net::IMAP recreated the raw original source string.
+      # Now, it grabs the raw encoded value using @str and @pos.  A future
+      # version may simply return the decoded astring value.  Although that is
+      # technically incompatible, it should almost never make a difference: all
+      # standard header field names are valid atoms:
+      #
+      # https://www.iana.org/assignments/message-headers/message-headers.xhtml
+      #
+      # Although RFC3501 allows any astring, RFC5322-valid header names are one
+      # or more of the printable US-ASCII characters, except SP and colon.  So
+      # empty string isn't valid, and literals aren't needed and should not be
+      # used.  This is explicitly unchanged by [I18N-HDRS] (RFC6532).
+      #
+      # RFC5233:
+      #     optional-field  =   field-name ":" unstructured CRLF
+      #     field-name      =   1*ftext
+      #     ftext           =   %d33-57 /          ; Printable US-ASCII
+      #                         %d59-126           ;  characters not including
+      #                                            ;  ":".
+      def header_fld_name
+        assert_no_lookahead
+        start = @pos
+        astring
+        @str[start...@pos - 1]
+      end
+
+      # mailbox-data    =  "FLAGS" SP flag-list / "LIST" SP mailbox-list /
+      #                    "LSUB" SP mailbox-list / "SEARCH" *(SP nz-number) /
+      #                    "STATUS" SP mailbox SP "(" [status-att-list] ")" /
+      #                    number SP "EXISTS" / number SP "RECENT"
+
+      def mailbox_data__flags
+        name = label("FLAGS")
+        SP!
+        UntaggedResponse.new(name, flag_list, @str)
+      end
+
+      def mailbox_data__list
+        name = label_in("LIST", "LSUB", "XLIST")
+        SP!
+        UntaggedResponse.new(name, mailbox_list, @str)
+      end
+      alias mailbox_data__lsub  mailbox_data__list
+      alias mailbox_data__xlist mailbox_data__list
+
+      # mailbox-list    = "(" [mbx-list-flags] ")" SP
+      #                    (DQUOTE QUOTED-CHAR DQUOTE / nil) SP mailbox
+      #                    [SP mbox-list-extended]
+      #             ; This is the list information pointed to by the ABNF
+      #             ; item "mailbox-data", which is defined above
       def mailbox_list
-        attr = flag_list
-        match(T_SPACE)
-        token = match(T_QUOTED, T_NIL)
-        if token.symbol == T_NIL
-          delim = nil
-        else
-          delim = token.value
-        end
-        match(T_SPACE)
-        name = astring
-        return MailboxList.new(attr, delim, name)
+        lpar; attr  = peek_rpar? ? [] : mbx_list_flags; rpar
+        SP!;  delim = nquoted
+        SP!;  name  = mailbox
+        # TODO: mbox-list-extended
+        MailboxList.new(attr, delim, name)
       end
 
       def getquota_response
@@ -813,183 +1408,229 @@ module Net
         quotaroots = []
         while true
           token = lookahead
-          break unless token.symbol == T_SPACE
-          shift_token
-          quotaroots.push(astring)
-        end
-        data = MailboxQuotaRoot.new(mailbox, quotaroots)
-        return UntaggedResponse.new(name, data, @str)
-      end
-
-      def getacl_response
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        mailbox = astring
-        data = []
-        token = lookahead
-        if token.symbol == T_SPACE
-          shift_token
-          while true
-            token = lookahead
-            case token.symbol
-            when T_CRLF
-              break
-            when T_SPACE
-              shift_token
-            end
-            user = astring
-            match(T_SPACE)
-            rights = astring
-            data.push(MailboxACLItem.new(user, rights, mailbox))
-          end
-        end
-        return UntaggedResponse.new(name, data, @str)
-      end
-
-      def search_response
-        token = match(T_ATOM)
-        name = token.value.upcase
-        token = lookahead
-        if token.symbol == T_SPACE
-          shift_token
-          data = []
-          while true
-            token = lookahead
-            case token.symbol
-            when T_CRLF
-              break
-            when T_SPACE
-              shift_token
-            when T_NUMBER
-              data.push(number)
-            when T_LPAR
-              # TODO: include the MODSEQ value in a response
-              shift_token
-              match(T_ATOM)
-              match(T_SPACE)
-              match(T_NUMBER)
-              match(T_RPAR)
-            end
-          end
-        else
-          data = []
-        end
-        return UntaggedResponse.new(name, data, @str)
-      end
-
-      def thread_response
-        token = match(T_ATOM)
-        name = token.value.upcase
-        token = lookahead
-
-        if token.symbol == T_SPACE
-          threads = []
-
-          while true
-            shift_token
-            token = lookahead
-
-            case token.symbol
-            when T_LPAR
-              threads << thread_branch(token)
-            when T_CRLF
-              break
-            end
-          end
-        else
-          # no member
-          threads = []
-        end
-
-        return UntaggedResponse.new(name, threads, @str)
-      end
-
-      def thread_branch(token)
-        rootmember = nil
-        lastmember = nil
-
-        while true
-          shift_token    # ignore first T_LPAR
-          token = lookahead
-
-          case token.symbol
-          when T_NUMBER
-            # new member
-            newmember = ThreadMember.new(number, [])
-            if rootmember.nil?
-              rootmember = newmember
-            else
-              lastmember.children << newmember
-            end
-            lastmember = newmember
-          when T_SPACE
-            # do nothing
-          when T_LPAR
-            if rootmember.nil?
-              # dummy member
-              lastmember = rootmember = ThreadMember.new(nil, [])
-            end
-
-            lastmember.children << thread_branch(token)
-          when T_RPAR
-            break
-          end
-        end
-
-        return rootmember
-      end
-
-      def status_response
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        mailbox = astring
-        match(T_SPACE)
-        match(T_LPAR)
-        attr = {}
-        while true
-          token = lookahead
-          case token.symbol
-          when T_RPAR
-            shift_token
-            break
-          when T_SPACE
-            shift_token
-          end
-          token = match(T_ATOM)
-          key = token.value.upcase
-          match(T_SPACE)
-          val = number
-          attr[key] = val
+          break unless token.symbol == T_SPACE
+          shift_token
+          quotaroots.push(astring)
         end
-        data = StatusData.new(mailbox, attr)
+        data = MailboxQuotaRoot.new(mailbox, quotaroots)
         return UntaggedResponse.new(name, data, @str)
       end
 
-      def capability_response
+      # acl-data        = "ACL" SP mailbox *(SP identifier SP rights)
+      def acl_data
         token = match(T_ATOM)
         name = token.value.upcase
         match(T_SPACE)
-        UntaggedResponse.new(name, capability_data, @str)
+        mailbox = astring
+        data = []
+        token = lookahead
+        if token.symbol == T_SPACE
+          shift_token
+          while true
+            token = lookahead
+            case token.symbol
+            when T_CRLF
+              break
+            when T_SPACE
+              shift_token
+            end
+            user = astring
+            match(T_SPACE)
+            rights = astring
+            data.push(MailboxACLItem.new(user, rights, mailbox))
+          end
+        end
+        return UntaggedResponse.new(name, data, @str)
       end
 
-      def capability_data
+      # RFC3501:
+      #   mailbox-data        = "SEARCH" *(SP nz-number) / ...
+      # RFC5256: SORT
+      #   sort-data           = "SORT" *(SP nz-number)
+      # RFC7162: CONDSTORE, QRESYNC
+      #   mailbox-data        =/ "SEARCH" [1*(SP nz-number) SP
+      #                          search-sort-mod-seq]
+      #   sort-data           = "SORT" [1*(SP nz-number) SP
+      #                           search-sort-mod-seq]
+      #                           ; Updates the SORT response from RFC 5256.
+      #   search-sort-mod-seq = "(" "MODSEQ" SP mod-sequence-value ")"
+      # RFC9051:
+      #   mailbox-data        = obsolete-search-response / ...
+      #   obsolete-search-response = "SEARCH" *(SP nz-number)
+      def mailbox_data__search
+        name = label_in("SEARCH", "SORT")
         data = []
-        while true
-          token = lookahead
-          case token.symbol
-          when T_CRLF, T_RBRA
-            break
-          when T_SPACE
-            shift_token
-            next
+        while _ = SP? && nz_number? do data << _ end
+        if lpar?
+          label("MODSEQ"); SP!
+          mod_sequence_value
+          rpar
+        end
+        UntaggedResponse.new(name, data, @str)
+      end
+      alias sort_data mailbox_data__search
+
+      # RFC5256: THREAD
+      #   thread-data     = "THREAD" [SP 1*thread-list]
+      def thread_data
+        name    = label("THREAD")
+        threads = []
+        if SP?
+          threads << thread_list while lookahead_thread_list?
+        end
+        UntaggedResponse.new(name, threads, @str)
+      end
+
+      alias lookahead_thread_list?   lookahead_lpar?
+      alias lookahead_thread_nested? lookahead_thread_list?
+
+      # RFC5256: THREAD
+      #   thread-list     = "(" (thread-members / thread-nested) ")"
+      def thread_list
+        lpar
+        thread = if lookahead_thread_nested?
+                   ThreadMember.new(nil, thread_nested)
+                 else
+                   thread_members
+                 end
+        rpar
+        thread
+      end
+
+      # RFC5256: THREAD
+      #   thread-members  = nz-number *(SP nz-number) [SP thread-nested]
+      def thread_members
+        members = []
+        members << nz_number # thread root
+        while SP?
+          case lookahead!(T_NUMBER, T_LPAR).symbol
+          when T_NUMBER then members << nz_number
+          else               nested = thread_nested; break
           end
-          data.push(atom.upcase)
         end
-        data
+        members.reverse.inject(nested || []) {|subthreads, number|
+          [ThreadMember.new(number, subthreads)]
+        }.first
+      end
+
+      # RFC5256: THREAD
+      #   thread-nested   = 2*thread-list
+      def thread_nested
+        nested = [thread_list, thread_list]
+        while lookahead_thread_list? do nested << thread_list end
+        nested
+      end
+
+      #   mailbox-data    =/ "STATUS" SP mailbox SP "(" [status-att-list] ")"
+      def mailbox_data__status
+        resp_name  = label("STATUS"); SP!
+        mbox_name  = mailbox;         SP!
+        lpar; attr = status_att_list; rpar
+        UntaggedResponse.new(resp_name, StatusData.new(mbox_name, attr), @str)
+      end
+
+      # RFC3501
+      #   status-att-list = status-att SP number *(SP status-att SP number)
+      # RFC4466, RFC9051, and RFC3501 Errata
+      #   status-att-list = status-att-val *(SP status-att-val)
+      def status_att_list
+        attrs = [status_att_val]
+        while SP? do attrs << status_att_val end
+        attrs.to_h
+      end
+
+      # RFC3501 Errata:
+      # status-att-val  = ("MESSAGES" SP number) / ("RECENT" SP number) /
+      #                   ("UIDNEXT" SP nz-number) / ("UIDVALIDITY" SP nz-number) /
+      #                   ("UNSEEN" SP number)
+      # RFC4466:
+      # status-att-val  = ("MESSAGES" SP number) /
+      #                   ("RECENT" SP number) /
+      #                   ("UIDNEXT" SP nz-number) /
+      #                   ("UIDVALIDITY" SP nz-number) /
+      #                   ("UNSEEN" SP number)
+      #                   ;; Extensions to the STATUS responses
+      #                   ;; should extend this production.
+      #                   ;; Extensions should use the generic
+      #                   ;; syntax defined by tagged-ext.
+      # RFC9051:
+      # status-att-val  = ("MESSAGES" SP number) /
+      #                   ("UIDNEXT" SP nz-number) /
+      #                   ("UIDVALIDITY" SP nz-number) /
+      #                   ("UNSEEN" SP number) /
+      #                   ("DELETED" SP number) /
+      #                   ("SIZE" SP number64)
+      #                     ; Extensions to the STATUS responses
+      #                     ; should extend this production.
+      #                     ; Extensions should use the generic
+      #                     ; syntax defined by tagged-ext.
+      # RFC7162:
+      # status-att-val      =/ "HIGHESTMODSEQ" SP mod-sequence-valzer
+      #                        ;; Extends non-terminal defined in [RFC4466].
+      #                        ;; Value 0 denotes that the mailbox doesn't
+      #                        ;; support persistent mod-sequences
+      #                        ;; as described in Section 3.1.2.2.
+      # RFC7889:
+      # status-att-val =/ "APPENDLIMIT" SP (number / nil)
+      #                 ;; status-att-val is defined in RFC 4466
+      # RFC8438:
+      # status-att-val =/ "SIZE" SP number64
+      # RFC8474:
+      # status-att-val =/ "MAILBOXID" SP "(" objectid ")"
+      #         ; follows tagged-ext production from [RFC4466]
+      def status_att_val
+        key = tagged_ext_label
+        SP!
+        val =
+          case key
+          when "MESSAGES"      then number              # RFC3501, RFC9051
+          when "UNSEEN"        then number              # RFC3501, RFC9051
+          when "DELETED"       then number              # RFC3501, RFC9051
+          when "UIDNEXT"       then nz_number           # RFC3501, RFC9051
+          when "UIDVALIDITY"   then nz_number           # RFC3501, RFC9051
+          when "RECENT"        then number              # RFC3501 (obsolete)
+          when "SIZE"          then number64            # RFC8483, RFC9051
+          when "MAILBOXID"     then parens__objectid    # RFC8474
+          else
+            number? || ExtensionData.new(tagged_ext_val)
+          end
+        [key, val]
+      end
+
+      # The presence of "IMAP4rev1" or "IMAP4rev2" is unenforced here.
+      # The grammar rule is used by both response-data and resp-text-code.
+      # But this method only returns UntaggedResponse (response-data).
+      #
+      # RFC3501:
+      #   capability-data  = "CAPABILITY" *(SP capability) SP "IMAP4rev1"
+      #                      *(SP capability)
+      # RFC9051:
+      #   capability-data  = "CAPABILITY" *(SP capability) SP "IMAP4rev2"
+      #                      *(SP capability)
+      def capability_data__untagged
+        UntaggedResponse.new label("CAPABILITY"), capability__list, @str
       end
 
+      # enable-data   = "ENABLED" *(SP capability)
+      def enable_data
+        UntaggedResponse.new label("ENABLED"), capability__list, @str
+      end
+
+      # As a workaround for buggy servers, allow a trailing SP:
+      #     *(SP capability) [SP]
+      def capability__list
+        list = []; while SP? && (capa = capability?) do list << capa end; list
+      end
+
+      alias resp_code__capability capability__list
+
+      # capability      = ("AUTH=" auth-type) / atom
+      #                     ; New capabilities MUST begin with "X" or be
+      #                     ; registered with IANA as standard or
+      #                     ; standards-track
+      alias capability  case_insensitive__atom
+      alias capability? case_insensitive__atom?
+
       def id_response
         token = match(T_ATOM)
         name = token.value.upcase
@@ -1019,147 +1660,181 @@ module Net
         end
       end
 
+      # namespace-response = "NAMESPACE" SP namespace
+      #                       SP namespace SP namespace
+      #                  ; The first Namespace is the Personal Namespace(s).
+      #                  ; The second Namespace is the Other Users'
+      #                  ; Namespace(s).
+      #                  ; The third Namespace is the Shared Namespace(s).
       def namespace_response
+        name = label("NAMESPACE")
         @lex_state = EXPR_DATA
-        token = lookahead
-        token = match(T_ATOM)
-        name = token.value.upcase
-        match(T_SPACE)
-        personal = namespaces
-        match(T_SPACE)
-        other = namespaces
-        match(T_SPACE)
-        shared = namespaces
+        data = Namespaces.new((SP!; namespace),
+                              (SP!; namespace),
+                              (SP!; namespace))
+        UntaggedResponse.new(name, data, @str)
+      ensure
         @lex_state = EXPR_BEG
-        data = Namespaces.new(personal, other, shared)
-        return UntaggedResponse.new(name, data, @str)
-      end
-
-      def namespaces
-        token = lookahead
-        # empty () is not allowed, so nil is functionally identical to empty.
-        data = []
-        if token.symbol == T_NIL
-          shift_token
-        else
-          match(T_LPAR)
-          loop do
-            data << namespace
-            break unless lookahead.symbol == T_SPACE
-            shift_token
-          end
-          match(T_RPAR)
-        end
-        data
       end
 
+      # namespace         = nil / "(" 1*namespace-descr ")"
       def namespace
-        match(T_LPAR)
-        prefix = match(T_QUOTED, T_LITERAL).value
-        match(T_SPACE)
-        delimiter = string
+        NIL? and return []
+        lpar
+        list = [namespace_descr]
+        list << namespace_descr until rpar?
+        list
+      end
+
+      # namespace-descr   = "(" string SP
+      #                        (DQUOTE QUOTED-CHAR DQUOTE / nil)
+      #                         [namespace-response-extensions] ")"
+      def namespace_descr
+        lpar
+        prefix     = string; SP!
+        delimiter  = nquoted # n.b: should only accept single char
         extensions = namespace_response_extensions
-        match(T_RPAR)
+        rpar
         Namespace.new(prefix, delimiter, extensions)
       end
 
+      # namespace-response-extensions = *namespace-response-extension
+      # namespace-response-extension = SP string SP
+      #                   "(" string *(SP string) ")"
       def namespace_response_extensions
         data = {}
-        token = lookahead
-        if token.symbol == T_SPACE
-          shift_token
-          name = match(T_QUOTED, T_LITERAL).value
+        while SP?
+          name = string; SP!
+          lpar
           data[name] ||= []
-          match(T_SPACE)
-          match(T_LPAR)
-          loop do
-            data[name].push match(T_QUOTED, T_LITERAL).value
-            break unless lookahead.symbol == T_SPACE
-            shift_token
-          end
-          match(T_RPAR)
+          data[name] << string
+          data[name] << string while SP?
+          rpar
         end
         data
       end
 
-      # text            = 1*TEXT-CHAR
-      # TEXT-CHAR       = <any CHAR except CR and LF>
+      #   TEXT-CHAR       = <any CHAR except CR and LF>
+      # RFC3501:
+      #   text            = 1*TEXT-CHAR
+      # RFC9051:
+      #   text            = 1*(TEXT-CHAR / UTF8-2 / UTF8-3 / UTF8-4)
+      #                     ; Non-ASCII text can only be returned
+      #                     ; after ENABLE IMAP4rev2 command
       def text
-        match(T_TEXT, lex_state: EXPR_TEXT).value
+        match_re(TEXT_REGEXP, "text")[0].force_encoding("UTF-8")
       end
 
-      # resp-text       = ["[" resp-text-code "]" SP] text
+      # an "accept" versiun of #text
+      def text?
+        accept_re(TEXT_REGEXP)&.[](0)&.force_encoding("UTF-8")
+      end
+
+      # RFC3501:
+      #   resp-text       = ["[" resp-text-code "]" SP] text
+      # RFC9051:
+      #   resp-text       = ["[" resp-text-code "]" SP] [text]
+      #
+      # We leniently re-interpret this as
+      #   resp-text       = ["[" resp-text-code "]" [SP [text]] / [text]
       def resp_text
-        token = match(T_LBRA, T_TEXT, lex_state: EXPR_RTEXT)
-        case token.symbol
-        when T_LBRA
-          code = resp_text_code
-          match(T_RBRA)
-          accept_space # violating RFC
-          ResponseText.new(code, text)
-        when T_TEXT
-          ResponseText.new(nil, token.value)
+        if lbra?
+          code = resp_text_code; rbra
+          ResponseText.new(code, SP? && text? || "")
+        else
+          ResponseText.new(nil, text? || "")
         end
       end
 
-      # See https://www.rfc-editor.org/errata/rfc3501
+      # RFC3501 (See https://www.rfc-editor.org/errata/rfc3501):
+      #   resp-text-code   = "ALERT" /
+      #                      "BADCHARSET" [SP "(" charset *(SP charset) ")" ] /
+      #                      capability-data / "PARSE" /
+      #                      "PERMANENTFLAGS" SP "(" [flag-perm *(SP flag-perm)] ")" /
+      #                      "READ-ONLY" / "READ-WRITE" / "TRYCREATE" /
+      #                      "UIDNEXT" SP nz-number / "UIDVALIDITY" SP nz-number /
+      #                      "UNSEEN" SP nz-number /
+      #                      atom [SP 1*<any TEXT-CHAR except "]">]
+      #   capability-data  = "CAPABILITY" *(SP capability) SP "IMAP4rev1"
+      #                      *(SP capability)
       #
-      # resp-text-code  = "ALERT" /
-      #                   "BADCHARSET" [SP "(" charset *(SP charset) ")" ] /
-      #                   capability-data / "PARSE" /
-      #                   "PERMANENTFLAGS" SP "("
-      #                   [flag-perm *(SP flag-perm)] ")" /
-      #                   "READ-ONLY" / "READ-WRITE" / "TRYCREATE" /
-      #                   "UIDNEXT" SP nz-number / "UIDVALIDITY" SP nz-number /
-      #                   "UNSEEN" SP nz-number /
-      #                   atom [SP 1*<any TEXT-CHAR except "]">]
+      # RFC5530:
+      #   resp-text-code  =/ "UNAVAILABLE" / "AUTHENTICATIONFAILED" /
+      #                     "AUTHORIZATIONFAILED" / "EXPIRED" /
+      #                     "PRIVACYREQUIRED" / "CONTACTADMIN" / "NOPERM" /
+      #                     "INUSE" / "EXPUNGEISSUED" / "CORRUPTION" /
+      #                     "SERVERBUG" / "CLIENTBUG" / "CANNOT" /
+      #                     "LIMIT" / "OVERQUOTA" / "ALREADYEXISTS" /
+      #                     "NONEXISTENT"
+      # RFC9051:
+      #   resp-text-code   = "ALERT" /
+      #                      "BADCHARSET" [SP "(" charset *(SP charset) ")" ] /
+      #                      capability-data / "PARSE" /
+      #                      "PERMANENTFLAGS" SP "(" [flag-perm *(SP flag-perm)] ")" /
+      #                      "READ-ONLY" / "READ-WRITE" / "TRYCREATE" /
+      #                      "UIDNEXT" SP nz-number / "UIDVALIDITY" SP nz-number /
+      #                      resp-code-apnd / resp-code-copy / "UIDNOTSTICKY" /
+      #                      "UNAVAILABLE" / "AUTHENTICATIONFAILED" /
+      #                      "AUTHORIZATIONFAILED" / "EXPIRED" /
+      #                      "PRIVACYREQUIRED" / "CONTACTADMIN" / "NOPERM" /
+      #                      "INUSE" / "EXPUNGEISSUED" / "CORRUPTION" /
+      #                      "SERVERBUG" / "CLIENTBUG" / "CANNOT" /
+      #                      "LIMIT" / "OVERQUOTA" / "ALREADYEXISTS" /
+      #                      "NONEXISTENT" / "NOTSAVED" / "HASCHILDREN" /
+      #                      "CLOSED" /
+      #                      "UNKNOWN-CTE" /
+      #                      atom [SP 1*<any TEXT-CHAR except "]">]
+      #   capability-data  = "CAPABILITY" *(SP capability) SP "IMAP4rev2"
+      #                      *(SP capability)
       #
-      # +UIDPLUS+ ABNF:: https://www.rfc-editor.org/rfc/rfc4315.html#section-4
-      #   resp-text-code  =/ resp-code-apnd / resp-code-copy / "UIDNOTSTICKY"
+      # RFC4315 (UIDPLUS), RFC9051 (IMAP4rev2):
+      #   resp-code-apnd   = "APPENDUID" SP nz-number SP append-uid
+      #   resp-code-copy   = "COPYUID" SP nz-number SP uid-set SP uid-set
+      #   resp-text-code   =/ resp-code-apnd / resp-code-copy / "UIDNOTSTICKY"
+      #
+      # RFC7162 (CONDSTORE):
+      #   resp-text-code   =/ "HIGHESTMODSEQ" SP mod-sequence-value /
+      #                       "NOMODSEQ" /
+      #                       "MODIFIED" SP sequence-set
+      #
+      # RFC8474: OBJECTID
+      #   resp-text-code   =/ "MAILBOXID" SP "(" objectid ")"
       def resp_text_code
-        token = match(T_ATOM)
-        name = token.value.upcase
-        case name
-        when /\A(?:ALERT|PARSE|READ-ONLY|READ-WRITE|TRYCREATE|NOMODSEQ)\z/n
-          result = ResponseCode.new(name, nil)
-        when /\A(?:BADCHARSET)\z/n
-          result = ResponseCode.new(name, charset_list)
-        when /\A(?:CAPABILITY)\z/ni
-          result = ResponseCode.new(name, capability_data)
-        when /\A(?:PERMANENTFLAGS)\z/n
-          match(T_SPACE)
-          result = ResponseCode.new(name, flag_list)
-        when /\A(?:UIDVALIDITY|UIDNEXT|UNSEEN)\z/n
-          match(T_SPACE)
-          result = ResponseCode.new(name, number)
-        when /\A(?:APPENDUID)\z/n
-          result = ResponseCode.new(name, resp_code_apnd__data)
-        when /\A(?:COPYUID)\z/n
-          result = ResponseCode.new(name, resp_code_copy__data)
-        else
-          token = lookahead
-          if token.symbol == T_SPACE
-            shift_token
-            token = match(T_TEXT, lex_state: EXPR_CTEXT)
-            result = ResponseCode.new(name, token.value)
+        name = resp_text_code__name
+        data =
+          case name
+          when "CAPABILITY"         then resp_code__capability
+          when "PERMANENTFLAGS"     then SP? ? flag_perm__list : []
+          when "UIDNEXT"            then SP!; nz_number
+          when "UIDVALIDITY"        then SP!; nz_number
+          when "UNSEEN"             then SP!; nz_number            # rev1 only
+          when "APPENDUID"          then SP!; resp_code_apnd__data # rev2, UIDPLUS
+          when "COPYUID"            then SP!; resp_code_copy__data # rev2, UIDPLUS
+          when "BADCHARSET"         then SP? ? charset__list : []
+          when "ALERT", "PARSE", "READ-ONLY", "READ-WRITE", "TRYCREATE",
+            "UNAVAILABLE", "AUTHENTICATIONFAILED", "AUTHORIZATIONFAILED",
+            "EXPIRED", "PRIVACYREQUIRED", "CONTACTADMIN", "NOPERM", "INUSE",
+            "EXPUNGEISSUED", "CORRUPTION", "SERVERBUG", "CLIENTBUG", "CANNOT",
+            "LIMIT", "OVERQUOTA", "ALREADYEXISTS", "NONEXISTENT", "CLOSED",
+            "NOTSAVED", "UIDNOTSTICKY", "UNKNOWN-CTE", "HASCHILDREN"
+          when "NOMODSEQ"           # CONDSTORE
+          when "MAILBOXID"          then SP!; parens__objectid     # RFC8474: OBJECTID
           else
-            result = ResponseCode.new(name, nil)
+            SP? and text_chars_except_rbra
           end
-        end
-        return result
+        ResponseCode.new(name, data)
       end
 
-      def charset_list
-        result = []
-        if accept(T_SPACE)
-          match(T_LPAR)
-          result << charset
-          while accept(T_SPACE)
-            result << charset
-          end
-          match(T_RPAR)
-        end
-        result
+      alias resp_text_code__name case_insensitive__atom
+
+      # 1*<any TEXT-CHAR except "]">
+      def text_chars_except_rbra
+        match_re(CTEXT_REGEXP, '1*<any TEXT-CHAR except "]">')[0]
+      end
+
+      # "(" charset *(SP charset) ")"
+      def charset__list
+        lpar; list = [charset]; while SP? do list << charset end; rpar; list
       end
 
       # already matched:  "APPENDUID"
@@ -1175,8 +1850,8 @@ module Net
       # match uid_set even if that returns a single-member array.
       #
       def resp_code_apnd__data
-        match(T_SPACE); validity = number
-        match(T_SPACE); dst_uids = uid_set # uniqueid ⊂ uid-set
+        validity = number; SP!
+        dst_uids = uid_set # uniqueid ⊂ uid-set
         UIDPlusData.new(validity, nil, dst_uids)
       end
 
@@ -1184,187 +1859,106 @@ module Net
       #
       # resp-code-copy  = "COPYUID" SP nz-number SP uid-set SP uid-set
       def resp_code_copy__data
-        match(T_SPACE); validity = number
-        match(T_SPACE); src_uids = uid_set
-        match(T_SPACE); dst_uids = uid_set
+        validity = number;  SP!
+        src_uids = uid_set; SP!
+        dst_uids = uid_set
         UIDPlusData.new(validity, src_uids, dst_uids)
       end
 
-      def address_list
-        token = lookahead
-        if token.symbol == T_NIL
-          shift_token
-          return nil
-        else
-          result = []
-          match(T_LPAR)
-          while true
-            token = lookahead
-            case token.symbol
-            when T_RPAR
-              shift_token
-              break
-            when T_SPACE
-              shift_token
-            end
-            result.push(address)
-          end
-          return result
-        end
-      end
-
-      ADDRESS_REGEXP = /\G\
-(?# 1: NAME     )(?:NIL|"((?:[^\x80-\xff\x00\r\n"\\]|\\["\\])*)") \
-(?# 2: ROUTE    )(?:NIL|"((?:[^\x80-\xff\x00\r\n"\\]|\\["\\])*)") \
-(?# 3: MAILBOX  )(?:NIL|"((?:[^\x80-\xff\x00\r\n"\\]|\\["\\])*)") \
-(?# 4: HOST     )(?:NIL|"((?:[^\x80-\xff\x00\r\n"\\]|\\["\\])*)")\
-\)/ni
-
+      ADDRESS_REGEXP = /\G
+        \( (?: NIL | #{Patterns::QUOTED_rev2} )  # 1: NAME
+        \s (?: NIL | #{Patterns::QUOTED_rev2} )  # 2: ROUTE
+        \s (?: NIL | #{Patterns::QUOTED_rev2} )  # 3: MAILBOX
+        \s (?: NIL | #{Patterns::QUOTED_rev2} )  # 4: HOST
+        \)
+      /nix
+
+      #   address         = "(" addr-name SP addr-adl SP addr-mailbox SP
+      #                     addr-host ")"
+      #   addr-adl        = nstring
+      #   addr-host       = nstring
+      #   addr-mailbox    = nstring
+      #   addr-name       = nstring
       def address
-        match(T_LPAR)
-        if @str.index(ADDRESS_REGEXP, @pos)
-          # address does not include literal.
-          @pos = $~.end(0)
-          name = $1
-          route = $2
-          mailbox = $3
-          host = $4
-          for s in [name, route, mailbox, host]
-            if s
-              s.gsub!(/\\(["\\])/n, "\\1")
-            end
-          end
-        else
-          name = nstring
-          match(T_SPACE)
-          route = nstring
-          match(T_SPACE)
-          mailbox = nstring
-          match(T_SPACE)
-          host = nstring
-          match(T_RPAR)
-        end
-        return Address.new(name, route, mailbox, host)
-      end
-
-      FLAG_REGEXP = /\
-(?# FLAG        )\\([^\x80-\xff(){ \x00-\x1f\x7f%"\\]+)|\
-(?# ATOM        )([^\x80-\xff(){ \x00-\x1f\x7f%*"\\]+)/n
-
+        if (match = accept_re(ADDRESS_REGEXP))
+          # note that "NIL" isn't captured by the regexp
+          name, route, mailbox, host = match.captures
+            .map { Patterns.unescape_quoted _1 }
+        else # address may include literals
+          lpar; name    = addr_name
+          SP!;  route   = addr_adl
+          SP!;  mailbox = addr_mailbox
+          SP!;  host    = addr_host
+          rpar
+        end
+        Address.new(name, route, mailbox, host)
+      end
+
+      alias addr_adl     nstring
+      alias addr_host    nstring
+      alias addr_mailbox nstring
+      alias addr_name    nstring
+
+      # flag-list       = "(" [flag *(SP flag)] ")"
       def flag_list
-        if @str.index(/\(([^)]*)\)/ni, @pos)
-          @pos = $~.end(0)
-          return $1.scan(FLAG_REGEXP).collect { |flag, atom|
-            if atom
-              atom
-            else
-              flag.capitalize.intern
-            end
-          }
-        else
-          parse_error("invalid flag list")
-        end
-      end
-
-      def nstring
-        token = lookahead
-        if token.symbol == T_NIL
-          shift_token
-          return nil
-        else
-          return string
-        end
+        match_re(Patterns::FLAG_LIST, "flag-list")[1]
+          .split(nil)
+          .map! { _1.start_with?("\\") ? _1[1..].capitalize.to_sym : _1 }
       end
 
-      def astring
-        token = lookahead
-        if string_token?(token)
-          return string
-        else
-          return astring_chars
-        end
+      #   "(" [flag-perm *(SP flag-perm)] ")"
+      def flag_perm__list
+        match_re(Patterns::FLAG_PERM_LIST, "PERMANENTFLAGS flag-perm list")[1]
+          .split(nil)
+          .map! { _1.start_with?("\\") ? _1[1..].capitalize.to_sym : _1 }
       end
 
-      def string
-        token = lookahead
-        if token.symbol == T_NIL
-          shift_token
-          return nil
-        end
-        token = match(T_QUOTED, T_LITERAL)
-        return token.value
+      # See Patterns::MBX_LIST_FLAGS
+      def mbx_list_flags
+        match_re(Patterns::MBX_LIST_FLAGS, "mbx-list-flags")[1]
+          .split(nil).map! { _1[1..].capitalize.to_sym }
       end
 
-      STRING_TOKENS = [T_QUOTED, T_LITERAL, T_NIL]
-
-      def string_token?(token)
-        return STRING_TOKENS.include?(token.symbol)
-      end
+      # See https://developers.google.com/gmail/imap/imap-extensions
+      def x_gm_label; accept(T_BSLASH) ? atom.capitalize.to_sym : astring end
 
-      def case_insensitive_string
-        token = lookahead
-        if token.symbol == T_NIL
-          shift_token
-          return nil
-        end
-        token = match(T_QUOTED, T_LITERAL)
-        return token.value.upcase
+      # See https://developers.google.com/gmail/imap/imap-extensions
+      def x_gm_labels
+        lpar; return [] if rpar?
+        labels = []
+        labels << x_gm_label
+        labels << x_gm_label while SP?
+        rpar
+        labels
       end
 
-      # atom            = 1*ATOM-CHAR
-      # ATOM-CHAR       = <any CHAR except atom-specials>
-      ATOM_TOKENS = [
-        T_ATOM,
-        T_NUMBER,
-        T_NIL,
-        T_LBRA,
-        T_PLUS
-      ]
-
-      def atom
-        -combine_adjacent(*ATOM_TOKENS)
-      end
+      # See https://www.rfc-editor.org/errata/rfc3501
+      #
+      # charset = atom / quoted
+      def charset; quoted? || atom end
 
-      # ASTRING-CHAR    = ATOM-CHAR / resp-specials
-      # resp-specials   = "]"
-      ASTRING_CHARS_TOKENS = [*ATOM_TOKENS, T_RBRA]
+      # RFC7162:
+      # mod-sequence-value  = 1*DIGIT
+      #                        ;; Positive unsigned 63-bit integer
+      #                        ;; (mod-sequence)
+      #                        ;; (1 <= n <= 9,223,372,036,854,775,807).
+      alias mod_sequence_value nz_number64
 
-      def astring_chars
-        combine_adjacent(*ASTRING_CHARS_TOKENS)
-      end
+      # RFC7162:
+      # permsg-modsequence  = mod-sequence-value
+      #                        ;; Per-message mod-sequence.
+      alias permsg_modsequence mod_sequence_value
 
-      def combine_adjacent(*tokens)
-        result = "".b
-        while token = accept(*tokens)
-          result << token.value
-        end
-        if result.empty?
-          parse_error('unexpected token %s (expected %s)',
-                      lookahead.symbol, args.join(" or "))
-        end
-        result
-      end
+      def parens__modseq; lpar; _ = permsg_modsequence; rpar; _ end
 
-      # See https://www.rfc-editor.org/errata/rfc3501
-      #
-      # charset = atom / quoted
-      def charset
-        if token = accept(T_QUOTED)
-          token.value
-        else
-          atom
-        end
-      end
+      # RFC8474:
+      # objectid = 1*255(ALPHA / DIGIT / "_" / "-")
+      #         ; characters in object identifiers are case
+      #         ; significant
+      alias objectid atom
 
-      def number
-        token = lookahead
-        if token.symbol == T_NIL
-          shift_token
-          return nil
-        end
-        token = match(T_NUMBER)
-        return token.value.to_i
-      end
+      def parens__objectid; lpar; _ = objectid; rpar; _ end
+      def nparens__objectid; NIL? ? nil : parens__objectid end
 
       # RFC-4315 (UIDPLUS) or RFC9051 (IMAP4rev2):
       #      uid-set         = (uniqueid / uid-range) *("," uid-set)
@@ -1393,64 +1987,15 @@ module Net
 
       SPACES_REGEXP = /\G */n
 
-      # This advances @pos directly so it's safe before changing @lex_state.
-      def accept_space
-        if @token
-          shift_token if @token.symbol == T_SPACE
-        elsif @str[@pos] == " "
-          @pos += 1
-        end
-      end
-
       # The RFC is very strict about this and usually we should be too.
       # But skipping spaces is usually a safe workaround for buggy servers.
       #
       # This advances @pos directly so it's safe before changing @lex_state.
       def accept_spaces
-        shift_token if @token&.symbol == T_SPACE
-        if @str.index(SPACES_REGEXP, @pos)
+        return false unless SP?
+        @str.index(SPACES_REGEXP, @pos) and
           @pos = $~.end(0)
-        end
-      end
-
-      def match(*args, lex_state: @lex_state)
-        if @token && lex_state != @lex_state
-          parse_error("invalid lex_state change to %s with unconsumed token",
-                      lex_state)
-        end
-        begin
-          @lex_state, original_lex_state = lex_state, @lex_state
-          token = lookahead
-          unless args.include?(token.symbol)
-            parse_error('unexpected token %s (expected %s)',
-                        token.symbol.id2name,
-                        args.collect {|i| i.id2name}.join(" or "))
-          end
-          shift_token
-          return token
-        ensure
-          @lex_state = original_lex_state
-        end
-      end
-
-      # like match, but does not raise error on failure.
-      #
-      # returns and shifts token on successful match
-      # returns nil and leaves @token unshifted on no match
-      def accept(*args)
-        token = lookahead
-        if args.include?(token.symbol)
-          shift_token
-          token
-        end
-      end
-
-      def lookahead
-        @token ||= next_token
-      end
-
-      def shift_token
-        @token = nil
+        true
       end
 
       def next_token
@@ -1461,38 +2006,46 @@ module Net
             if $1
               return Token.new(T_SPACE, $+)
             elsif $2
-              return Token.new(T_NIL, $+)
-            elsif $3
-              return Token.new(T_NUMBER, $+)
+              len = $+.to_i
+              val = @str[@pos, len]
+              @pos += len
+              return Token.new(T_LITERAL8, val)
+            elsif $3 && $7
+              # greedily match ATOM, prefixed with NUMBER, NIL, or PLUS.
+              return Token.new(T_ATOM, $3)
             elsif $4
-              return Token.new(T_ATOM, $+)
+              return Token.new(T_NIL, $+)
             elsif $5
-              return Token.new(T_QUOTED,
-                               $+.gsub(/\\(["\\])/n, "\\1"))
+              return Token.new(T_NUMBER, $+)
             elsif $6
+              return Token.new(T_PLUS, $+)
+            elsif $8
+              # match ATOM, without a NUMBER, NIL, or PLUS prefix
+              return Token.new(T_ATOM, $+)
+            elsif $9
+              return Token.new(T_QUOTED, Patterns.unescape_quoted($+))
+            elsif $10
               return Token.new(T_LPAR, $+)
-            elsif $7
+            elsif $11
               return Token.new(T_RPAR, $+)
-            elsif $8
+            elsif $12
               return Token.new(T_BSLASH, $+)
-            elsif $9
+            elsif $13
               return Token.new(T_STAR, $+)
-            elsif $10
+            elsif $14
               return Token.new(T_LBRA, $+)
-            elsif $11
+            elsif $15
               return Token.new(T_RBRA, $+)
-            elsif $12
+            elsif $16
               len = $+.to_i
               val = @str[@pos, len]
               @pos += len
               return Token.new(T_LITERAL, val)
-            elsif $13
-              return Token.new(T_PLUS, $+)
-            elsif $14
+            elsif $17
               return Token.new(T_PERCENT, $+)
-            elsif $15
+            elsif $18
               return Token.new(T_CRLF, $+)
-            elsif $16
+            elsif $19
               return Token.new(T_EOF, $+)
             else
               parse_error("[Net::IMAP BUG] BEG_REGEXP is invalid")
@@ -1511,8 +2064,7 @@ module Net
             elsif $3
               return Token.new(T_NUMBER, $+)
             elsif $4
-              return Token.new(T_QUOTED,
-                               $+.gsub(/\\(["\\])/n, "\\1"))
+              return Token.new(T_QUOTED, Patterns.unescape_quoted($+))
             elsif $5
               len = $+.to_i
               val = @str[@pos, len]
@@ -1529,63 +2081,11 @@ module Net
             @str.index(/\S*/n, @pos)
             parse_error("unknown token - %s", $&.dump)
           end
-        when EXPR_TEXT
-          if @str.index(TEXT_REGEXP, @pos)
-            @pos = $~.end(0)
-            if $1
-              return Token.new(T_TEXT, $+)
-            else
-              parse_error("[Net::IMAP BUG] TEXT_REGEXP is invalid")
-            end
-          else
-            @str.index(/\S*/n, @pos)
-            parse_error("unknown token - %s", $&.dump)
-          end
-        when EXPR_RTEXT
-          if @str.index(RTEXT_REGEXP, @pos)
-            @pos = $~.end(0)
-            if $1
-              return Token.new(T_LBRA, $+)
-            elsif $2
-              return Token.new(T_TEXT, $+)
-            else
-              parse_error("[Net::IMAP BUG] RTEXT_REGEXP is invalid")
-            end
-          else
-            @str.index(/\S*/n, @pos)
-            parse_error("unknown token - %s", $&.dump)
-          end
-        when EXPR_CTEXT
-          if @str.index(CTEXT_REGEXP, @pos)
-            @pos = $~.end(0)
-            if $1
-              return Token.new(T_TEXT, $+)
-            else
-              parse_error("[Net::IMAP BUG] CTEXT_REGEXP is invalid")
-            end
-          else
-            @str.index(/\S*/n, @pos) #/
-            parse_error("unknown token - %s", $&.dump)
-          end
         else
           parse_error("invalid @lex_state - %s", @lex_state.inspect)
         end
       end
 
-      def parse_error(fmt, *args)
-        if IMAP.debug
-          $stderr.printf("@str: %s\n", @str.dump)
-          $stderr.printf("@pos: %d\n", @pos)
-          $stderr.printf("@lex_state: %s\n", @lex_state)
-          if @token
-            $stderr.printf("@token.symbol: %s\n", @token.symbol)
-            $stderr.printf("@token.value: %s\n", @token.value.inspect)
-          end
-        end
-        raise ResponseParseError, format(fmt, *args)
-      end
     end
-
   end
-
 end