net-imap 0.3.7 → 0.4.4

data/lib/net/imap/response_data.rb

@@ -55,17 +55,54 @@ module Net
 
     # Net::IMAP::IgnoredResponse represents intentionally ignored responses.
     #
-    # This includes untagged response "NOOP" sent by eg. Zimbra to avoid some
-    # clients to close the connection.
+    # This includes untagged response "NOOP" sent by eg. Zimbra to avoid
+    # some clients to close the connection.
     #
     # It matches no IMAP standard.
+    class IgnoredResponse < UntaggedResponse
+    end
+
+    # **Note:** This represents an intentionally _unstable_ API.  Where
+    # instances of this class are returned, future releases may return a
+    # different (incompatible) object <em>without deprecation or warning</em>.
     #
-    class IgnoredResponse < Struct.new(:raw_data)
+    # Net::IMAP::UnparsedData represents data for unknown response types or
+    # unknown extensions to response types without a well-defined extension
+    # grammar.
+    #
+    # See also: UnparsedNumericResponseData
+    class UnparsedData < Struct.new(:unparsed_data)
       ##
-      # method: raw_data
-      # :call-seq: raw_data -> string
+      # method: unparsed_data
+      # :call-seq: unparsed_data -> string
       #
-      # The raw response data.
+      # The unparsed data
+    end
+
+    # **Note:** This represents an intentionally _unstable_ API.  Where
+    # instances of this class are returned, future releases may return a
+    # different (incompatible) object <em>without deprecation or warning</em>.
+    #
+    # Net::IMAP::UnparsedNumericResponseData represents data for unhandled
+    # response types with a numeric prefix.  See the documentation for #number.
+    #
+    # See also: UnparsedData
+    class UnparsedNumericResponseData < Struct.new(:number, :unparsed_data)
+      ##
+      # method: number
+      # :call-seq: number -> integer
+      #
+      # Returns a numeric response data prefix, when available.
+      #
+      # Many response types are prefixed with a non-negative #number.  For
+      # message data, #number may represent a sequence number or a UID.  For
+      # mailbox data, #number may represent a message count.
+
+      ##
+      # method: unparsed_data
+      # :call-seq: unparsed_data -> string
+      #
+      # The unparsed data, not including #number or UntaggedResponse#name.
     end
 
     # Net::IMAP::TaggedResponse represents tagged responses.
@@ -108,6 +145,9 @@ module Net
     # UntaggedResponse#data when the response type is a "condition" ("OK", "NO",
     # "BAD", "PREAUTH", or "BYE").
     class ResponseText < Struct.new(:code, :text)
+      # Used to avoid an allocation when ResponseText is empty
+      EMPTY = new(nil, "").freeze
+
       ##
       # method: code
       # :call-seq: code -> ResponseCode or nil
@@ -891,13 +931,6 @@ module Net
     #                   should use BodyTypeBasic.
     # BodyTypeMultipart:: for <tt>multipart/*</tt> parts
     #
-    # ==== Deprecated BodyStructure classes
-    # The following classes represent invalid server responses or parser bugs:
-    # BodyTypeExtension:: parser bug: used for <tt>message/*</tt> where
-    #                     BodyTypeBasic should have been used.
-    # BodyTypeAttachment:: server bug: some servers sometimes return the
-    #                      "Content-Disposition: attachment" data where the
-    #                      entire body structure for a message part is expected.
     module BodyStructure
     end
 
@@ -914,6 +947,7 @@ module Net
                                      :param, :content_id,
                                      :description, :encoding, :size,
                                      :md5, :disposition, :language,
+                                     :location,
                                      :extension)
       include BodyStructure
 
@@ -1049,6 +1083,7 @@ module Net
                                     :description, :encoding, :size,
                                     :lines,
                                     :md5, :disposition, :language,
+                                    :location,
                                     :extension)
       include BodyStructure
 
@@ -1094,6 +1129,7 @@ module Net
                                        :description, :encoding, :size,
                                        :envelope, :body, :lines,
                                        :md5, :disposition, :language,
+                                       :location,
                                        :extension)
       include BodyStructure
 
@@ -1126,36 +1162,41 @@ module Net
       end
     end
 
-    # === WARNING
-    # BodyTypeAttachment represents a <tt>body-fld-dsp</tt> that is
-    # incorrectly in a position where the IMAP4rev1 grammar expects a nested
-    # +body+ structure.
+    # BodyTypeAttachment is not used and will be removed in an upcoming release.
     #
-    # >>>
-    #   \IMAP body structures are parenthesized lists and assign their fields
-    #   positionally, so missing fields change the intepretation of all
-    #   following fields.  Buggy \IMAP servers sometimes leave fields missing
-    #   rather than empty, which inevitably confuses parsers.
-    #   BodyTypeAttachment was an attempt to parse a common type of buggy body
-    #   structure without crashing.
-    #
-    #   Currently, when Net::IMAP::ResponseParser sees "attachment" as the first
-    #   entry in a <tt>body-type-1part</tt>, which is where the MIME type should
-    #   be, it uses BodyTypeAttachment to capture the rest.  "attachment" is not
-    #   a valid MIME type, but _is_ a common <tt>Content-Disposition</tt>.  What
-    #   might have happened was that buggy server could not parse the message
-    #   (which might have been incorrectly formatted) and output a
-    #   <tt>body-type-dsp</tt> where a Net::IMAP::ResponseParser expected to see
-    #   a +body+.
-    #
-    # A future release will replace this, probably with a ContentDisposition
-    # nested inside another body structure object, maybe BodyTypeBasic, or
-    # perhaps a new body structure class that represents any unparsable body
-    # structure.
+    # === Bug Analysis
+    #
+    # \IMAP body structures are parenthesized lists and assign their fields
+    # positionally, so missing fields change the intepretation of all
+    # following fields.  Additionally, different body types have a different
+    # number of required fields, followed by optional "extension" fields.
+    #
+    # BodyTypeAttachment was previously returned when a "message/rfc822" part,
+    # which should be sent as <tt>body-type-msg</tt> with ten required fields,
+    # was actually sent as a <tt>body-type-basic</tt> with _seven_ required
+    # fields.
+    #
+    #   basic => type, subtype, param, id, desc, enc, octets, md5=nil,  dsp=nil, lang=nil, loc=nil, *ext
+    #   msg   => type, subtype, param, id, desc, enc, octets, envelope, body,    lines,    md5=nil, ...
+    #
+    # Normally, +envelope+ and +md5+ are incompatible, but Net::IMAP leniently
+    # allowed buggy servers to send +NIL+ for +envelope+.  As a result, when a
+    # server sent a <tt>message/rfc822</tt> part with +NIL+ for +md5+ and a
+    # non-<tt>NIL</tt> +dsp+, Net::IMAP mis-interpreted the
+    # <tt>Content-Disposition</tt> as if it were a strange body type.  In all
+    # reported cases, the <tt>Content-Disposition</tt> was "attachment", so
+    # BodyTypeAttachment was created as the workaround.
+    #
+    # === Current behavior
+    #
+    # When interpreted strictly, +envelope+ and +md5+ are incompatible.  So the
+    # current parsing algorithm peeks ahead after it has recieved the seventh
+    # body field.  If the next token is not the start of an +envelope+, we assume
+    # the server has incorrectly sent us a <tt>body-type-basic</tt> and return
+    # BodyTypeBasic.  As a result, what was previously BodyTypeMessage#body =>
+    # BodyTypeAttachment is now BodyTypeBasic#disposition => ContentDisposition.
     #
     class BodyTypeAttachment < Struct.new(:dsp_type, :_unused_, :param)
-      include BodyStructure
-
       # *invalid for BodyTypeAttachment*
       def media_type
         warn(<<~WARN, uplevel: 1)
@@ -1190,11 +1231,14 @@ module Net
       end
     end
 
+    deprecate_constant :BodyTypeAttachment
+
     # Net::IMAP::BodyTypeMultipart represents body structures of messages and
     # message parts, when <tt>Content-Type</tt> is <tt>multipart/*</tt>.
     class BodyTypeMultipart < Struct.new(:media_type, :subtype,
                                          :parts,
                                          :param, :disposition, :language,
+                                         :location,
                                          :extension)
       include BodyStructure
 
@@ -1265,23 +1309,24 @@ module Net
       end
     end
 
-    # === WARNING:
+    # === Obsolete
+    # BodyTypeExtension is not used and will be removed in an upcoming release.
+    #
     # >>>
-    #   BodyTypeExtension is (incorrectly) used for <tt>message/*</tt> parts
+    #   BodyTypeExtension was (incorrectly) used for <tt>message/*</tt> parts
     #   (besides <tt>message/rfc822</tt>, which correctly uses BodyTypeMessage).
     #
-    # A future release will replace this class with:
-    # * BodyTypeMessage for <tt>message/rfc822</tt> and <tt>message/global</tt>
-    # * BodyTypeBasic for any other <tt>message/*</tt>
+    #   Net::IMAP now (correctly) parses all message types (other than
+    #   <tt>message/rfc822</tt> or <tt>message/global</tt>) as BodyTypeBasic.
     class BodyTypeExtension < Struct.new(:media_type, :subtype,
                                          :params, :content_id,
                                          :description, :encoding, :size)
-      include BodyStructure
-
       def multipart?
         return false
       end
     end
 
+    deprecate_constant :BodyTypeExtension
+
   end
 end

data/lib/net/imap/response_parser/parser_utils.rb

@@ -0,0 +1,240 @@
+# frozen_string_literal: true
+
+module Net
+  class IMAP < Protocol
+    class ResponseParser
+      # basic utility methods for parsing.
+      #
+      # (internal API, subject to change)
+      module ParserUtils # :nodoc:
+
+        module Generator
+
+          LOOKAHEAD = "(@token ||= next_token)"
+          SHIFT_TOKEN = "(@token = nil)"
+
+          # we can skip lexer for single character matches, as a shortcut
+          def def_char_matchers(name, char, token)
+            byte = char.ord
+            match_name = name.match(/\A[A-Z]/) ? "#{name}!" : name
+            char = char.dump
+            class_eval <<~RUBY, __FILE__, __LINE__ + 1
+              # frozen_string_literal: true
+
+              # force use of #next_token; no string peeking
+              def lookahead_#{name}?
+                #{LOOKAHEAD}&.symbol == #{token}
+              end
+
+              # use token or string peek
+              def peek_#{name}?
+                @token ? @token.symbol == #{token} : @str.getbyte(@pos) == #{byte}
+              end
+
+              # like accept(token_symbols); returns token or nil
+              def #{name}?
+                if @token&.symbol == #{token}
+                  #{SHIFT_TOKEN}
+                  #{char}
+                elsif !@token && @str.getbyte(@pos) == #{byte}
+                  @pos += 1
+                  #{char}
+                end
+              end
+
+              # like match(token_symbols); returns token or raises parse_error
+              def #{match_name}
+                if @token&.symbol == #{token}
+                  #{SHIFT_TOKEN}
+                  #{char}
+                elsif !@token && @str.getbyte(@pos) == #{byte}
+                  @pos += 1
+                  #{char}
+                else
+                  parse_error("unexpected %s (expected %p)",
+                              @token&.symbol || @str[@pos].inspect, #{char})
+                end
+              end
+            RUBY
+          end
+
+          # TODO: move coersion to the token.value method?
+          def def_token_matchers(name, *token_symbols, coerce: nil, send: nil)
+            match_name = name.match(/\A[A-Z]/) ? "#{name}!" : name
+
+            if token_symbols.size == 1
+              token   = token_symbols.first
+              matcher = "token&.symbol == %p" % [token]
+              desc    = token
+            else
+              matcher = "%p.include? token&.symbol" % [token_symbols]
+              desc    = token_symbols.join(" or ")
+            end
+
+            value = "(token.value)"
+            value = coerce.to_s + value   if coerce
+            value = [value, send].join(".") if send
+
+            raise_parse_error = <<~RUBY
+              parse_error("unexpected %s (expected #{desc})", token&.symbol)
+            RUBY
+
+            class_eval <<~RUBY, __FILE__, __LINE__ + 1
+              # frozen_string_literal: true
+
+              # lookahead version of match, returning the value
+              def lookahead_#{name}!
+                token = #{LOOKAHEAD}
+                if #{matcher}
+                  #{value}
+                else
+                  #{raise_parse_error}
+                end
+              end
+
+              def #{name}?
+                token = #{LOOKAHEAD}
+                if #{matcher}
+                  #{SHIFT_TOKEN}
+                  #{value}
+                end
+              end
+
+              def #{match_name}
+                token = #{LOOKAHEAD}
+                if #{matcher}
+                  #{SHIFT_TOKEN}
+                  #{value}
+                else
+                  #{raise_parse_error}
+                end
+              end
+            RUBY
+          end
+
+        end
+
+        private
+
+        # TODO: after checking the lookahead, use a regexp for remaining chars.
+        # That way a loop isn't needed.
+        def combine_adjacent(*tokens)
+          result = "".b
+          while token = accept(*tokens)
+            result << token.value
+          end
+          if result.empty?
+            parse_error('unexpected token %s (expected %s)',
+                        lookahead.symbol, tokens.join(" or "))
+          end
+          result
+        end
+
+        def match(*args)
+          token = lookahead
+          unless args.include?(token.symbol)
+            parse_error('unexpected token %s (expected %s)',
+                        token.symbol.id2name,
+                        args.collect {|i| i.id2name}.join(" or "))
+          end
+          shift_token
+          token
+        end
+
+        # like match, but does not raise error on failure.
+        #
+        # returns and shifts token on successful match
+        # returns nil and leaves @token unshifted on no match
+        def accept(*args)
+          token = lookahead
+          if args.include?(token.symbol)
+            shift_token
+            token
+          end
+        end
+
+        # To be used conditionally:
+        #   assert_no_lookahead if Net::IMAP.debug
+        def assert_no_lookahead
+          @token.nil? or
+            parse_error("assertion failed: expected @token.nil?, actual %s: %p",
+                        @token.symbol, @token.value)
+        end
+
+        # like accept, without consuming the token
+        def lookahead?(*symbols)
+          @token if symbols.include?((@token ||= next_token)&.symbol)
+        end
+
+        def lookahead
+          @token ||= next_token
+        end
+
+        # like match, without consuming the token
+        def lookahead!(*args)
+          if args.include?((@token ||= next_token)&.symbol)
+            @token
+          else
+            parse_error('unexpected token %s (expected %s)',
+                        @token&.symbol, args.join(" or "))
+          end
+        end
+
+        def peek_str?(str)
+          assert_no_lookahead if Net::IMAP.debug
+          @str[@pos, str.length] == str
+        end
+
+        def peek_re(re)
+          assert_no_lookahead if Net::IMAP.debug
+          re.match(@str, @pos)
+        end
+
+        def accept_re(re)
+          assert_no_lookahead if Net::IMAP.debug
+          re.match(@str, @pos) and @pos = $~.end(0)
+          $~
+        end
+
+        def match_re(re, name)
+          assert_no_lookahead if Net::IMAP.debug
+          if re.match(@str, @pos)
+            @pos = $~.end(0)
+            $~
+          else
+            parse_error("invalid #{name}")
+          end
+        end
+
+        def shift_token
+          @token = nil
+        end
+
+        def parse_error(fmt, *args)
+          msg = format(fmt, *args)
+          if IMAP.debug
+            local_path = File.dirname(__dir__)
+            tok = @token ? "%s: %p" % [@token.symbol, @token.value] : "nil"
+            warn "%s %s: %s"        % [self.class, __method__, msg]
+            warn "  tokenized : %s" % [@str[...@pos].dump]
+            warn "  remaining : %s" % [@str[@pos..].dump]
+            warn "  @lex_state: %s" % [@lex_state]
+            warn "  @pos      : %d" % [@pos]
+            warn "  @token    : %s" % [tok]
+            caller_locations(1..20).each_with_index do |cloc, idx|
+              next unless cloc.path&.start_with?(local_path)
+              warn "  caller[%2d]: %-30s (%s:%d)" % [
+                idx,
+                cloc.base_label,
+                File.basename(cloc.path, ".rb"),
+                cloc.lineno
+              ]
+            end
+          end
+          raise ResponseParseError, msg
+        end
+
+      end
+    end
+  end
+end