net-imap 0.6.2 → 0.6.4

data/lib/net/imap/errors.rb

@@ -51,7 +51,196 @@ module Net
     end
 
     # Error raised when a response from the server is non-parsable.
+    #
+    # NOTE: Parser attributes are provided for debugging and inspection only.
+    # Their names and semantics may change incompatibly in any release.
     class ResponseParseError < Error
+      # returns "" for all highlights
+      ESC_NO_HL = Hash.new("").freeze
+      private_constant :ESC_NO_HL
+
+      # Translates hash[:"/foo"] to hash[:reset] when hash.key?(:foo), else ""
+      #
+      # TODO: DRY this up with Config::AttrTypeCoercion.safe
+      if defined?(::Ractor.shareable_proc)
+        default_highlight = Ractor.shareable_proc {|hash, key|
+          %r{\A/(.+)} =~ key && hash.key?($1.to_sym) ? hash[:reset] : ""
+        }
+      else
+        default_highlight = nil.instance_eval { Proc.new {|hash, key|
+          %r{\A/(.+)} =~ key && hash.key?($1.to_sym) ? hash[:reset] : ""
+        } }
+        ::Ractor.make_shareable(default_highlight) if defined?(::Ractor)
+      end
+
+      # ANSI highlights, but no colors
+      ESC_NO_COLOR = Hash.new(&default_highlight).update(
+        reset: "\e[m",
+        val:   "\e[1m",   # bold
+        alt:   "\e[1;4m", # bold and underlined
+        sym:   "\e[1m",   # bold
+        label: "\e[1m",   # bold
+      ).freeze
+      private_constant :ESC_NO_COLOR
+
+      # ANSI highlights, with color
+      ESC_COLORS = Hash.new(&default_highlight).update(
+        reset: "\e[m",
+        key:   "\e[95m",      # bright magenta
+        idx:   "\e[34m",      # blue
+        val:   "\e[36;40m",   # cyan on black (to ensure contrast)
+        alt:   "\e[1;33;40m", # bold; yellow on black
+        sym:   "\e[33;40m",   # yellow on black
+        label: "\e[1m",       # bold
+        nil:   "\e[35m",      # magenta
+      ).freeze
+      private_constant :ESC_COLORS
+
+      # Net::IMAP::ResponseParser, unless a custom parser produced the error.
+      attr_reader :parser_class
+
+      # The full raw response string which was being parsed.
+      attr_reader :string
+
+      # The parser's byte position in #string when the error was raised.
+      #
+      # _NOTE:_ This attribute is provided for debugging and inspection only.
+      # Its name and semantics may change incompatibly in any release.
+      attr_reader :pos
+
+      # The parser's lex state
+      #
+      # _NOTE:_ This attribute is provided for debugging and inspection only.
+      # Its name and semantics may change incompatibly in any release.
+      attr_reader :lex_state
+
+      # The last lexed token
+      #
+      # May be +nil+ when the parser has accepted the last token and peeked at
+      # the next byte without generating a token.
+      #
+      # _NOTE:_ This attribute is provided for debugging and inspection only.
+      # Its name and semantics may change incompatibly in any release.
+      attr_reader :token
+
+      def initialize(message = "unspecified parse error",
+                     parser_class: Net::IMAP::ResponseParser,
+                     parser_state: nil,
+                     string:    parser_state&.at(0), # see ParserUtils#parser_state
+                     lex_state: parser_state&.at(1), # see ParserUtils#parser_state
+                     pos:       parser_state&.at(2), # see ParserUtils#parser_state
+                     token:     parser_state&.at(3)) # see ParserUtils#parser_state
+        @parser_class = parser_class
+        @string    = string
+        @pos       = pos
+        @lex_state = lex_state
+        @token     = token
+        super(message)
+      end
+
+      # When +parser_state+ is true, debug info about the parser state is
+      # included.  Defaults to the value of Net::IMAP.debug.
+      #
+      # When +parser_backtrace+ is true, a simplified backtrace is included,
+      # containing only frames for methods in parser_class (since ruby 3.4) or
+      # which have "net/imap/response_parser" in the path (before ruby 3.4).
+      # Most parser method names are based on rules in the IMAP grammar.
+      #
+      # When +highlight+ is not explicitly set, highlights may be enabled
+      # automatically, based on +TERM+ and +FORCE_COLOR+ environment variables.
+      #
+      # By default, +highlight+ uses colors from the basic ANSI palette.  When
+      # +highlight_no_color+ is true or the +NO_COLOR+ environment variable is
+      # not empty, only monochromatic highlights are used: bold, underline, etc.
+      def detailed_message(parser_state: Net::IMAP.debug,
+                           parser_backtrace: false,
+                           highlight: default_highlight_from_env,
+                           highlight_no_color: (ENV["NO_COLOR"] || "") != "",
+                           **)
+        return super unless parser_state || parser_backtrace
+        msg = super.dup
+        esc = !highlight ? ESC_NO_HL : highlight_no_color ? ESC_NO_COLOR : ESC_COLORS
+        hl  = ->str { str % esc }
+        val = ->str, val { hl[val.nil? ? "%{nil}%%p%{/nil}" : str] % val }
+        if parser_state && (string || pos || lex_state || token)
+          msg << hl["\n  %{key}processed %{/key}: "] << val["%{val}%%p%{/val}", processed_string]
+          msg << hl["\n  %{key}remaining %{/key}: "] << val["%{alt}%%p%{/alt}", remaining_string]
+          msg << hl["\n  %{key}pos       %{/key}: "] << val["%{val}%%p%{/val}", pos]
+          msg << hl["\n  %{key}lex_state %{/key}: "] << val["%{sym}%%p%{/sym}", lex_state]
+          msg << hl["\n  %{key}token     %{/key}: "] << val[
+            "%{sym}%%<symbol>p%{/sym} => %{val}%%<value>p%{/val}", token&.to_h
+          ]
+        end
+        if parser_backtrace
+          normalized_parser_backtrace.each do |idx, path, lineno, label, base_label|
+            msg << "\n  %s: %s (%s:%d)" % [
+              hl["%{key}caller[%{/key}%{idx}%%2d%{/idx}%{key}]%{/key}"] % idx,
+              hl["%{label}%%-30s%{/label}"] % base_label,
+              File.basename(path, ".rb"), lineno
+            ]
+          end
+        end
+        msg
+      rescue => error
+        msg ||= super.dup
+        msg << "\n  BUG in %s#%s: %s" % [self.class, __method__,
+                                         error.detailed_message]
+        msg
+      end
+
+      def processed_string = string && pos && string[...pos]
+      def remaining_string = string && pos && string[pos..]
+
+      # Returns true when all attributes are equal, except for #backtrace and
+      # #backtrace_locations which are replaced with #parser_methods.  This
+      # allows deserialized errors to be compared.
+      def ==(other)
+        return false if self.class != other.class
+        methods       = parser_methods
+        other_methods = other.parser_methods
+        message     == other.message &&
+          methods   == other_methods &&
+          string    == other.string &&
+          pos       == other.pos &&
+          lex_state == other.lex_state &&
+          token     == other.token
+      end
+
+      # Lists the methods (from #backtrace_locations or #backtrace) called on
+      # parser_class (since ruby 3.4) or which have "net/imap/response_parser"
+      # in the path (before ruby 3.4).  Most parser method names are based on
+      # rules in the IMAP grammar.
+      def parser_methods = normalized_parser_backtrace.map(&:last)
+
+      private
+
+      def normalized_parser_backtrace
+        normalize_backtrace
+          .take_while {|_, _, _, _, base_label| base_label != "parse" }
+          .reject {|_, _, _, _, base_label| base_label.nil? }
+          .reject {|_, _, _, _, base_label| base_label.include? "parse_error" }
+          .select {|_, path, _, label, _|
+            if label.include?("#") # => Class#method, since ruby 3.4
+              label.include?(parser_class.name)
+            else
+              path.include?("net/imap/response_parser")
+            end
+          }
+      end
+
+      def normalize_backtrace
+        (backtrace_locations&.each_with_index&.map {|loc, idx|
+          [idx, loc.path, loc.lineno, loc.label, loc.base_label]
+        } || backtrace&.each_with_index&.map {|bt, idx|
+          [idx, *bt.match(/\A(\S+):(\d+):in [`'](.*?([\w]+[?!]?))'\z/)&.captures]
+        } || [])
+      end
+
+      def default_highlight_from_env
+        (ENV["FORCE_COLOR"] || "") !~ /\A(?:0|)\z/ ||
+          (ENV["TERM"] || "") !~ /\A(?:dumb|unknown|)\z/i
+      end
+
     end
 
     # Superclass of all errors used to encapsulate "fail" responses

data/lib/net/imap/response_data.rb

@@ -75,9 +75,18 @@ module Net
     #
     # Net::IMAP::UnparsedData represents data for unknown response types or
     # unknown extensions to response types without a well-defined extension
-    # grammar.
+    # grammar.  UnparsedData represents the portion of the response which the
+    # parser has skipped over, without attempting to parse it.
     #
-    # See also: UnparsedNumericResponseData, ExtensionData, IgnoredResponse
+    #    parser = Net::IMAP::ResponseParser.new
+    #    response = parser.parse "* X-UNKNOWN-TYPE can't parse this\r\n"
+    #    response => Net::IMAP::UntaggedResponse(
+    #      name: "X-UNKNOWN-TYPE",
+    #      data: Net::IMAP::UnparsedData(unparsed_data: "can't parse this"),
+    #    )
+    #
+    # See also: UnparsedNumericResponseData, ExtensionData, IgnoredResponse,
+    # InvalidParseData.
     class UnparsedData < Struct.new(:unparsed_data)
       ##
       # method: unparsed_data
@@ -86,6 +95,61 @@ module Net
       # The unparsed data
     end
 
+    # **Note:** This represents an intentionally _unstable_ API.  Where
+    # instances of this class are returned, future releases may return a
+    # different (incompatible) object <em>without deprecation or warning</em>.
+    #
+    # When the response parser encounters a recoverable error,
+    # Net::IMAP::InvalidParseData represents that portion of the response which
+    # could not be parsed, allowing the parser to parse the remainder of the
+    # response.  InvalidParseData is always associated with a ResponseParseError
+    # which has been rescued.
+    #
+    # This could be caused by a malformed server response, by a bug in
+    # Net::IMAP::ResponseParser, or by an unsupported extension to the response
+    # syntax.  For example, if a server supports +UIDPLUS+, but sends an invalid
+    # +COPYUID+ response code:
+    #
+    #    parser = Net::IMAP::ResponseParser.new
+    #    parsed = parser.parse "* OK [COPYUID 701  ] copied one message\r\n"
+    #    parsed => {
+    #      data: Net::IMAP::ResponseText(
+    #        code: Net::IMAP::ResponseCode(
+    #          name: "COPYUID",
+    #          data: Net::IMAP::InvalidParseData(
+    #            parse_error: Net::IMAP::ResponseParseError,
+    #            unparsed_data: "701  ",
+    #            parsed_data: nil,
+    #          )
+    #        )
+    #      )
+    #    }
+    #
+    # In this example, although <tt>[COPYUID 701  ]</tt> uses valid syntax for a
+    # _generic_ ResponseCode, it is _invalid_ syntax for a +COPYUID+ response
+    # code.
+    #
+    # See also: UnparsedData, ExtensionData
+    class InvalidParseData < Data.define(:parse_error, :unparsed_data, :parsed_data)
+      ##
+      # method: parse_error
+      # :call-seq: parse_error -> ResponseParseError
+      #
+      # Returns the rescued ResponseParseError.
+
+      ##
+      # method: unparsed_data
+      # :call-seq: unparsed_data -> string
+      #
+      # Returns the raw string which was skipped over by the parser.
+
+      ##
+      # method: parsed_data
+      #
+      # May return a partial parse result for unparsed_data, which had already
+      # been parsed before the parse_error.
+    end
+
     # **Note:** This represents an intentionally _unstable_ API.  Where
     # instances of this class are returned, future releases may return a
     # different (incompatible) object <em>without deprecation or warning</em>.
@@ -93,7 +157,17 @@ module Net
     # Net::IMAP::UnparsedNumericResponseData represents data for unhandled
     # response types with a numeric prefix.  See the documentation for #number.
     #
-    # See also: UnparsedData, ExtensionData, IgnoredResponse
+    #    parser = Net::IMAP::ResponseParser.new
+    #    response = parser.parse "* 123 X-UNKNOWN-TYPE can't parse this\r\n"
+    #    response => Net::IMAP::UntaggedResponse(
+    #      name: "X-UNKNOWN-TYPE",
+    #      data: Net::IMAP::UnparsedNumericData(
+    #        number: 123,
+    #        unparsed_data: "can't parse this"
+    #      ),
+    #    )
+    #
+    # See also: UnparsedData, ExtensionData, IgnoredResponse, InvalidParseData
     class UnparsedNumericResponseData < Struct.new(:number, :unparsed_data)
       ##
       # method: number
@@ -306,6 +380,14 @@ module Net
     #   because the server doesn't allow deletion of mailboxes with children.
     #   #data is +nil+.
     #
+    # === <tt>QUOTA=RES-*</tt> response codes
+    # See {[RFC9208]}[https://www.rfc-editor.org/rfc/rfc9208.html#section-4.3].
+    # * +OVERQUOTA+ (also in RFC5530[https://www.rfc-editor.org/rfc/rfc5530]),
+    #   with a tagged +NO+ response to an +APPEND+/+COPY+/+MOVE+ command when
+    #   the command would put the target mailbox over any quota, and with an
+    #   untagged +NO+ when a mailbox exceeds a soft quota (which may be caused
+    #   be external events).  #data is +nil+.
+    #
     # === +CONDSTORE+ extension
     # See {[RFC7162]}[https://www.rfc-editor.org/rfc/rfc7162.html].
     # * +NOMODSEQ+, when selecting a mailbox that does not support
@@ -324,9 +406,10 @@ module Net
     #
     # Response codes are backwards compatible:  Servers are allowed to send new
     # response codes even if the client has not enabled the extension that
-    # defines them.  When Net::IMAP does not know how to parse response
-    # code text, #data returns the unparsed string.
-    #
+    # defines them.  When ResponseParser does not know how to parse the response
+    # code data, #data may return the unparsed string, ExtensionData, or
+    # UnparsedData.  When ResponseParser attempts but fails to parse the
+    # response code data, #data returns InvalidParseData.
     class ResponseCode < Struct.new(:name, :data)
       ##
       # method: name
@@ -341,8 +424,13 @@ module Net
       #
       # Returns the parsed response code data, e.g: an array of capabilities
       # strings, an array of character set strings, a list of permanent flags,
-      # an Integer, etc.  The response #code determines what form the response
-      # code data can take.
+      # an Integer, etc.  The response #name determines what form the response
+      # code #data can take.
+      #
+      # When ResponseParser does not know how to parse the response code data,
+      # #data may return the unparsed string, ExtensionData, or UnparsedData.
+      # When ResponseParser attempts but fails to parse the response code data,
+      # #data returns InvalidParseData.
     end
 
     # MailboxList represents the data of an untagged +LIST+ response, for a
@@ -383,14 +471,23 @@ module Net
     # and MailboxQuota objects.
     #
     # == Required capability
+    #
     # Requires +QUOTA+ [RFC2087[https://www.rfc-editor.org/rfc/rfc2087]]
-    # capability.
+    # or <tt>QUOTA=RES-STORAGE</tt>
+    # [RFC9208[https://www.rfc-editor.org/rfc/rfc9208]] capability.
     class MailboxQuota < Struct.new(:mailbox, :usage, :quota)
       ##
       # method: mailbox
       # :call-seq: mailbox -> string
       #
-      # The mailbox with the associated quota.
+      # The quota root with the associated quota.
+      #
+      # NOTE: this was mistakenly named "mailbox".  But the quota root's name may
+      # differ from the mailbox.  A single quota root may cover multiple
+      # mailboxes, and a single mailbox may be governed by multiple quota roots.
+
+      # The quota root with the associated quota.
+      alias quota_root mailbox
 
       ##
       # method: usage
@@ -402,7 +499,7 @@ module Net
       # method: quota
       # :call-seq: quota -> Integer
       #
-      # Quota limit imposed on the mailbox.
+      # Storage limit imposed on the mailbox.
       #
     end
 

data/lib/net/imap/response_parser/parser_utils.rb

@@ -215,29 +215,20 @@ module Net
           @token = nil
         end
 
-        def parse_error(fmt, *args)
-          msg = format(fmt, *args)
-          if config.debug?
-            local_path = File.dirname(__dir__)
-            tok = @token ? "%s: %p" % [@token.symbol, @token.value] : "nil"
-            warn "%s %s: %s"        % [self.class, __method__, msg]
-            warn "  tokenized : %s" % [@str[...@pos].dump]
-            warn "  remaining : %s" % [@str[@pos..].dump]
-            warn "  @lex_state: %s" % [@lex_state]
-            warn "  @pos      : %d" % [@pos]
-            warn "  @token    : %s" % [tok]
-            caller_locations(1..20).each_with_index do |cloc, idx|
-              next unless cloc.path&.start_with?(local_path)
-              warn "  caller[%2d]: %-30s (%s:%d)" % [
-                idx,
-                cloc.base_label,
-                File.basename(cloc.path, ".rb"),
-                cloc.lineno
-              ]
-            end
-          end
-          raise ResponseParseError, msg
-        end
+        def parse_error(fmt, *args) = raise exception format(fmt, *args)
+
+        def exception(message) = ResponseParseError.new(
+          message, parser_state:, parser_class: self.class
+        )
+
+        # This can be used to backtrack after a parse error, and re-attempt to
+        # parse using a fallback.
+        #
+        # NOTE: Reckless backtracking could lead to O(n²) situations, so this
+        # should very rarely be used.  Ideally, fallbacks should not backtrack.
+        def restore_state(state) = (@lex_state, @pos, @token = state)
+        def current_state        = [@lex_state, @pos, @token]
+        def parser_state         = [@str, *current_state]
 
       end
     end

data/lib/net/imap/response_parser.rb

@@ -38,6 +38,11 @@ module Net
         @lex_state = EXPR_BEG
         @token = nil
         return response
+      rescue ResponseParseError => error
+        if config.debug?
+          warn error.detailed_message(parser_state: true, parser_backtrace: true)
+        end
+        raise
       end
 
       private
@@ -686,6 +691,8 @@ module Net
         CRLF!
         EOF!
         resp
+      rescue SystemStackError
+        parse_error("response recursion too deep")
       end
 
       # RFC3501 & RFC9051:
@@ -1883,12 +1890,17 @@ module Net
       # We leniently re-interpret this as
       #   resp-text       = ["[" resp-text-code "]" [SP [text]] / [text]
       def resp_text
-        if lbra?
-          code = resp_text_code; rbra
-          ResponseText.new(code, SP? && text? || "")
-        else
-          ResponseText.new(nil, text? || "")
+        begin
+          state = current_state
+          if lbra?
+            code = resp_text_code; rbra
+            return ResponseText.new(code, SP? && text? || "")
+          end
+        rescue ResponseParseError => error
+          raise if /\buid-set\b/i.match? error.message
+          restore_state state
         end
+        ResponseText.new(nil, text? || "")
       end
 
       # RFC3501 (See https://www.rfc-editor.org/errata/rfc3501):
@@ -1951,6 +1963,7 @@ module Net
       #   resp-text-code   =/ "UIDREQUIRED"
       def resp_text_code
         name = resp_text_code__name
+        state = current_state
         data =
           case name
           when "CAPABILITY"         then resp_code__capability
@@ -1973,8 +1986,18 @@ module Net
           when "MAILBOXID"          then SP!; parens__objectid     # RFC8474: OBJECTID
           when "UIDREQUIRED"        then                           # RFC9586: UIDONLY
           else
+            state = nil # don't backtrack
             SP? and text_chars_except_rbra
           end
+        peek_rbra? or
+          parse_error("expected resp-text-code %p to be complete", name)
+        ResponseCode.new(name, data)
+      rescue Net::IMAP::ResponseParseError => parse_error
+        raise unless state
+        raise if parse_error.message.include?("uid-set")
+        restore_state state
+        unparsed_data = SP? && text_chars_except_rbra
+        data = InvalidParseData[parse_error:, unparsed_data:, parsed_data: data]
         ResponseCode.new(name, data)
       end
 
@@ -2017,13 +2040,18 @@ module Net
         CopyUID(validity, src_uids, dst_uids)
       end
 
+      PARSER_PATH = File.expand_path(__FILE__).delete_suffix(".rb")
+
       # TODO: remove this code in the v0.6.0 release
       def DeprecatedUIDPlus(validity, src_uids = nil, dst_uids)
         return unless config.parser_use_deprecated_uidplus_data
+        uplevel = caller_locations
+          .find_index { !_1.path.start_with?(PARSER_PATH) }
+          &.succ
         warn("#{Config}#parser_use_deprecated_uidplus_data is ignored " \
              "since v0.6.0.  Disable this warning by setting " \
              "config.parser_use_deprecated_uidplus_data = false.",
-             category: :deprecated, uplevel: 9)
+             category: :deprecated, uplevel:)
         nil
       end
 

data/lib/net/imap/response_reader.rb

@@ -8,51 +8,60 @@ module Net
 
       def initialize(client, sock)
         @client, @sock = client, sock
+        # cached config
+        @max_response_size = nil
+        # response buffer state
+        @buff = @literal_size = nil
       end
 
       def read_response_buffer
+        @max_response_size = client.max_response_size
         @buff = String.new
         catch :eof do
           while true
+            guard_response_too_large!
             read_line
-            break unless (@literal_size = get_literal_size)
+            # check before allocating memory for literal
+            guard_response_too_large!
+            break unless literal_size
             read_literal
           end
         end
         buff
       ensure
-        @buff = nil
+        @buff = @literal_size = nil
       end
 
       private
 
+      # cached config
+      attr_reader :max_response_size
+
+      # response buffer state
       attr_reader :buff, :literal_size
 
       def bytes_read          = buff.bytesize
       def empty?              = buff.empty?
-      def done?               = line_done? && !get_literal_size
+      def done?               = line_done? && !literal_size
       def line_done?          = buff.end_with?(CRLF)
-      def get_literal_size    = /\{(\d+)\}\r\n\z/n =~ buff && $1.to_i
+
+      def get_literal_size(buff)
+        buff.end_with?("}\r\n") && buff.rindex(/\{(\d+)\}\r\n\z/n) && $1.to_i
+      end
 
       def read_line
-        buff << (@sock.gets(CRLF, read_limit) or throw :eof)
-        max_response_remaining! unless line_done?
+        line = (@sock.gets(CRLF, max_response_remaining) or throw :eof)
+        @literal_size = get_literal_size(line)
+        buff << line
       end
 
       def read_literal
-        # check before allocating memory for literal
-        max_response_remaining!
         literal = String.new(capacity: literal_size)
-        buff << (@sock.read(read_limit(literal_size), literal) or throw :eof)
+        buff << (@sock.read(literal_size, literal) or throw :eof)
       ensure
         @literal_size = nil
       end
 
-      def read_limit(limit = nil)
-        [limit, max_response_remaining!].compact.min
-      end
-
-      def max_response_size      = client.max_response_size
       def max_response_remaining = max_response_size &.- bytes_read
       def response_too_large?    = max_response_size &.< min_response_size
       def min_response_size      = bytes_read + min_response_remaining
@@ -61,8 +70,8 @@ module Net
         empty? ? 3 : done? ? 0 : (literal_size || 0) + 2
       end
 
-      def max_response_remaining!
-        return max_response_remaining unless response_too_large?
+      def guard_response_too_large!
+        return unless response_too_large?
         raise ResponseTooLargeError.new(
           max_response_size:, bytes_read:, literal_size:,
         )