net-imap 0.3.7 → 0.4.1

data/lib/net/imap/authenticators.rb

@@ -1,68 +1,37 @@
 # frozen_string_literal: true
 
-# Registry for SASL authenticators used by Net::IMAP.
+# Backward compatible delegators from Net::IMAP to Net::IMAP::SASL.
 module Net::IMAP::Authenticators
 
-  # Adds an authenticator for Net::IMAP#authenticate to use.  +mechanism+ is the
-  # {SASL mechanism}[https://www.iana.org/assignments/sasl-mechanisms/sasl-mechanisms.xhtml]
-  # implemented by +authenticator+ (for instance, <tt>"PLAIN"</tt>).
-  #
-  # The +authenticator+ must respond to +#new+ (or #call), receiving the
-  # authenticator configuration and return a configured authentication session.
-  # The authenticator session must respond to +#process+, receiving the server's
-  # challenge and returning the client's response.
-  #
-  # See PlainAuthenticator, XOauth2Authenticator, and DigestMD5Authenticator for
-  # examples.
-  def add_authenticator(auth_type, authenticator)
-    authenticators[auth_type] = authenticator
+  # Deprecated.  Use Net::IMAP::SASL.add_authenticator instead.
+  def add_authenticator(...)
+    warn(
+      "%s.%s is deprecated.  Use %s.%s instead." % [
+        Net::IMAP, __method__, Net::IMAP::SASL, __method__
+      ],
+      uplevel: 1
+    )
+    Net::IMAP::SASL.add_authenticator(...)
   end
 
-  # :call-seq:
-  #   authenticator(mechanism, ...)                            -> authenticator
-  #   authenticator(mech, *creds, **props) {|prop, auth| val } -> authenticator
-  #   authenticator(mechanism, authnid, creds, authzid=nil)    -> authenticator
-  #   authenticator(mechanism, **properties)                   -> authenticator
-  #   authenticator(mechanism) {|propname, authctx| value }    -> authenticator
-  #
-  # Builds a new authentication session context for +mechanism+.
-  #
-  # [Note]
-  #   This method is intended for internal use by connection protocol code only.
-  #   Protocol client users should see refer to their client's documentation,
-  #   e.g. Net::IMAP#authenticate for Net::IMAP.
-  #
-  # The call signatures documented for this method are recommendations for
-  # authenticator implementors.  All arguments (other than +mechanism+) are
-  # forwarded to the registered authenticator's +#new+ (or +#call+) method, and
-  # each authenticator must document its own arguments.
-  #
-  # The returned object represents a single authentication exchange and <em>must
-  # not</em> be reused for multiple authentication attempts.
-  def authenticator(mechanism, *authargs, **properties, &callback)
-    authenticator = authenticators.fetch(mechanism.upcase) do
-      raise ArgumentError, 'unknown auth type - "%s"' % mechanism
-    end
-    if authenticator.respond_to?(:new)
-      authenticator.new(*authargs, **properties, &callback)
-    else
-      authenticator.call(*authargs, **properties, &callback)
-    end
-  end
-
-  private
-
-  def authenticators
-    @authenticators ||= {}
+  # Deprecated.  Use Net::IMAP::SASL.authenticator instead.
+  def authenticator(...)
+    warn(
+      "%s.%s is deprecated.  Use %s.%s instead." % [
+        Net::IMAP, __method__, Net::IMAP::SASL, __method__
+      ],
+      uplevel: 1
+    )
+    Net::IMAP::SASL.authenticator(...)
   end
 
+  Net::IMAP.extend self
 end
 
-Net::IMAP.extend Net::IMAP::Authenticators
+class Net::IMAP
+  PlainAuthenticator = SASL::PlainAuthenticator # :nodoc:
+  deprecate_constant :PlainAuthenticator
 
-require_relative "authenticators/plain"
-
-require_relative "authenticators/login"
-require_relative "authenticators/cram_md5"
-require_relative "authenticators/digest_md5"
-require_relative "authenticators/xoauth2"
+  XOauth2Authenticator = SASL::XOAuth2Authenticator # :nodoc:
+  deprecate_constant :XOauth2Authenticator
+end

data/lib/net/imap/command_data.rb

@@ -52,13 +52,20 @@ module Net
     end
 
     def send_string_data(str, tag = nil)
-      case str
-      when ""
+      if str.empty?
         put_string('""')
-      when /[\x80-\xff\r\n]/n
-        # literal
+      elsif str.match?(/[\r\n]/n)
+        # literal, because multiline
         send_literal(str, tag)
-      when /[(){ \x00-\x1f\x7f%*"\\]/n
+      elsif !str.ascii_only?
+        if @utf8_strings
+          # quoted string
+          send_quoted_string(str)
+        else
+          # literal, because of non-ASCII bytes
+          send_literal(str, tag)
+        end
+      elsif str.match?(/[(){ \x00-\x1f\x7f%*"\\]/n)
         # quoted string
         send_quoted_string(str)
       else
@@ -67,7 +74,7 @@ module Net
     end
 
     def send_quoted_string(str)
-      put_string('"' + str.gsub(/["\\]/n, "\\\\\\&") + '"')
+      put_string('"' + str.gsub(/["\\]/, "\\\\\\&") + '"')
     end
 
     def send_literal(str, tag = nil)

data/lib/net/imap/deprecated_client_options.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+module Net
+  class IMAP < Protocol
+
+    # This module handles deprecated arguments to various Net::IMAP methods.
+    module DeprecatedClientOptions
+
+      # :call-seq:
+      #   Net::IMAP.new(host, **options) # standard keyword options
+      #   Net::IMAP.new(host, options)   # obsolete hash options
+      #   Net::IMAP.new(host, port)      # obsolete port argument
+      #   Net::IMAP.new(host, port, usessl, certs = nil, verify = true) # deprecated SSL arguments
+      #
+      # Translates Net::IMAP.new arguments for backward compatibility.
+      #
+      # ==== Obsolete arguments
+      #
+      # Using obsolete arguments does not a warning.  Obsolete arguments will be
+      # deprecated by a future release.
+      #
+      # If a second positional argument is given and it is a hash (or is
+      # convertable via +#to_hash+), it is converted to keyword arguments.
+      #
+      #     # Obsolete:
+      #     Net::IMAP.new("imap.example.com", options_hash)
+      #     # Use instead:
+      #     Net::IMAP.new("imap.example.com", **options_hash)
+      #
+      # If a second positional argument is given and it is not a hash, it is
+      # converted to the +port+ keyword argument.
+      #     # Obsolete:
+      #     Net::IMAP.new("imap.example.com", 114433)
+      #     # Use instead:
+      #     Net::IMAP.new("imap.example.com", port: 114433)
+      #
+      # ==== Deprecated arguments
+      #
+      # Using deprecated arguments prints a warning.  Convert to keyword
+      # arguments to avoid the warning.  Deprecated arguments will be removed in
+      # a future release.
+      #
+      # If +usessl+ is false, +certs+, and +verify+ are ignored.  When it true,
+      # all three arguments are converted to the +ssl+ keyword argument.
+      # Without +certs+ or +verify+, it is converted to <tt>ssl: true</tt>.
+      #     # DEPRECATED:
+      #     Net::IMAP.new("imap.example.com", nil, true) # => prints a warning
+      #     # Use instead:
+      #     Net::IMAP.new("imap.example.com", ssl: true)
+      #
+      # When +certs+ is a path to a directory, it is converted to <tt>ca_path:
+      # certs</tt>.
+      #     # DEPRECATED:
+      #     Net::IMAP.new("imap.example.com", nil, true, "/path/to/certs") # => prints a warning
+      #     # Use instead:
+      #     Net::IMAP.new("imap.example.com", ssl: {ca_path: "/path/to/certs"})
+      #
+      # When +certs+ is a path to a file, it is converted to <tt>ca_file:
+      # certs</tt>.
+      #     # DEPRECATED:
+      #     Net::IMAP.new("imap.example.com", nil, true, "/path/to/cert.pem") # => prints a warning
+      #     # Use instead:
+      #     Net::IMAP.new("imap.example.com", ssl: {ca_file: "/path/to/cert.pem"})
+      #
+      # When +verify+ is +false+, it is converted to <tt>verify_mode:
+      # OpenSSL::SSL::VERIFY_NONE</tt>.
+      #     # DEPRECATED:
+      #     Net::IMAP.new("imap.example.com", nil, true, nil, false) # => prints a warning
+      #     # Use instead:
+      #     Net::IMAP.new("imap.example.com", ssl: {verify_mode: OpenSSL::SSL::VERIFY_NONE})
+      #
+      def initialize(host, port_or_options = nil, *deprecated, **options)
+        if port_or_options.nil? && deprecated.empty?
+          super host, **options
+        elsif options.any?
+          # Net::IMAP.new(host, *__invalid__, **options)
+          raise ArgumentError, "Do not combine deprecated and keyword arguments"
+        elsif port_or_options.respond_to?(:to_hash) and deprecated.any?
+          # Net::IMAP.new(host, options, *__invalid__)
+          raise ArgumentError, "Do not use deprecated SSL params with options hash"
+        elsif port_or_options.respond_to?(:to_hash)
+          super host, **Hash.try_convert(port_or_options)
+        elsif deprecated.empty?
+          super host, port: port_or_options
+        elsif deprecated.shift
+          warn "DEPRECATED: Call Net::IMAP.new with keyword options", uplevel: 1
+          super host, port: port_or_options, ssl: create_ssl_params(*deprecated)
+        else
+          warn "DEPRECATED: Call Net::IMAP.new with keyword options", uplevel: 1
+          super host, port: port_or_options, ssl: false
+        end
+      end
+
+      # :call-seq:
+      #   starttls(**options) # standard
+      #   starttls(options = {}) # obsolete
+      #   starttls(certs = nil, verify = true) # deprecated
+      #
+      # Translates Net::IMAP#starttls arguments for backward compatibility.
+      #
+      # Support for +certs+ and +verify+ will be dropped in a future release.
+      #
+      # See ::new for interpretation of +certs+ and +verify+.
+      def starttls(*deprecated, **options)
+        if deprecated.empty?
+          super(**options)
+        elsif options.any?
+          # starttls(*__invalid__, **options)
+          raise ArgumentError, "Do not combine deprecated and keyword options"
+        elsif deprecated.first.respond_to?(:to_hash) && deprecated.length > 1
+          # starttls(*__invalid__, **options)
+          raise ArgumentError, "Do not use deprecated verify param with options hash"
+        elsif deprecated.first.respond_to?(:to_hash)
+          super(**Hash.try_convert(deprecated.first))
+        else
+          warn "DEPRECATED: Call Net::IMAP#starttls with keyword options", uplevel: 1
+          super(**create_ssl_params(*deprecated))
+        end
+      end
+
+      private
+
+      def create_ssl_params(certs = nil, verify = true)
+        params = {}
+        if certs
+          if File.file?(certs)
+            params[:ca_file] = certs
+          elsif File.directory?(certs)
+            params[:ca_path] = certs
+          end
+        end
+        params[:verify_mode] =
+          verify ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+        params
+      end
+
+    end
+  end
+end

data/lib/net/imap/response_data.rb

@@ -891,13 +891,6 @@ module Net
     #                   should use BodyTypeBasic.
     # BodyTypeMultipart:: for <tt>multipart/*</tt> parts
     #
-    # ==== Deprecated BodyStructure classes
-    # The following classes represent invalid server responses or parser bugs:
-    # BodyTypeExtension:: parser bug: used for <tt>message/*</tt> where
-    #                     BodyTypeBasic should have been used.
-    # BodyTypeAttachment:: server bug: some servers sometimes return the
-    #                      "Content-Disposition: attachment" data where the
-    #                      entire body structure for a message part is expected.
     module BodyStructure
     end
 
@@ -914,6 +907,7 @@ module Net
                                      :param, :content_id,
                                      :description, :encoding, :size,
                                      :md5, :disposition, :language,
+                                     :location,
                                      :extension)
       include BodyStructure
 
@@ -1049,6 +1043,7 @@ module Net
                                     :description, :encoding, :size,
                                     :lines,
                                     :md5, :disposition, :language,
+                                    :location,
                                     :extension)
       include BodyStructure
 
@@ -1094,6 +1089,7 @@ module Net
                                        :description, :encoding, :size,
                                        :envelope, :body, :lines,
                                        :md5, :disposition, :language,
+                                       :location,
                                        :extension)
       include BodyStructure
 
@@ -1126,36 +1122,41 @@ module Net
       end
     end
 
-    # === WARNING
-    # BodyTypeAttachment represents a <tt>body-fld-dsp</tt> that is
-    # incorrectly in a position where the IMAP4rev1 grammar expects a nested
-    # +body+ structure.
+    # BodyTypeAttachment is not used and will be removed in an upcoming release.
     #
-    # >>>
-    #   \IMAP body structures are parenthesized lists and assign their fields
-    #   positionally, so missing fields change the intepretation of all
-    #   following fields.  Buggy \IMAP servers sometimes leave fields missing
-    #   rather than empty, which inevitably confuses parsers.
-    #   BodyTypeAttachment was an attempt to parse a common type of buggy body
-    #   structure without crashing.
-    #
-    #   Currently, when Net::IMAP::ResponseParser sees "attachment" as the first
-    #   entry in a <tt>body-type-1part</tt>, which is where the MIME type should
-    #   be, it uses BodyTypeAttachment to capture the rest.  "attachment" is not
-    #   a valid MIME type, but _is_ a common <tt>Content-Disposition</tt>.  What
-    #   might have happened was that buggy server could not parse the message
-    #   (which might have been incorrectly formatted) and output a
-    #   <tt>body-type-dsp</tt> where a Net::IMAP::ResponseParser expected to see
-    #   a +body+.
-    #
-    # A future release will replace this, probably with a ContentDisposition
-    # nested inside another body structure object, maybe BodyTypeBasic, or
-    # perhaps a new body structure class that represents any unparsable body
-    # structure.
+    # === Bug Analysis
+    #
+    # \IMAP body structures are parenthesized lists and assign their fields
+    # positionally, so missing fields change the intepretation of all
+    # following fields.  Additionally, different body types have a different
+    # number of required fields, followed by optional "extension" fields.
+    #
+    # BodyTypeAttachment was previously returned when a "message/rfc822" part,
+    # which should be sent as <tt>body-type-msg</tt> with ten required fields,
+    # was actually sent as a <tt>body-type-basic</tt> with _seven_ required
+    # fields.
+    #
+    #   basic => type, subtype, param, id, desc, enc, octets, md5=nil,  dsp=nil, lang=nil, loc=nil, *ext
+    #   msg   => type, subtype, param, id, desc, enc, octets, envelope, body,    lines,    md5=nil, ...
+    #
+    # Normally, +envelope+ and +md5+ are incompatible, but Net::IMAP leniently
+    # allowed buggy servers to send +NIL+ for +envelope+.  As a result, when a
+    # server sent a <tt>message/rfc822</tt> part with +NIL+ for +md5+ and a
+    # non-<tt>NIL</tt> +dsp+, Net::IMAP mis-interpreted the
+    # <tt>Content-Disposition</tt> as if it were a strange body type.  In all
+    # reported cases, the <tt>Content-Disposition</tt> was "attachment", so
+    # BodyTypeAttachment was created as the workaround.
+    #
+    # === Current behavior
+    #
+    # When interpreted strictly, +envelope+ and +md5+ are incompatible.  So the
+    # current parsing algorithm peeks ahead after it has recieved the seventh
+    # body field.  If the next token is not the start of an +envelope+, we assume
+    # the server has incorrectly sent us a <tt>body-type-basic</tt> and return
+    # BodyTypeBasic.  As a result, what was previously BodyTypeMessage#body =>
+    # BodyTypeAttachment is now BodyTypeBasic#disposition => ContentDisposition.
     #
     class BodyTypeAttachment < Struct.new(:dsp_type, :_unused_, :param)
-      include BodyStructure
-
       # *invalid for BodyTypeAttachment*
       def media_type
         warn(<<~WARN, uplevel: 1)
@@ -1190,11 +1191,14 @@ module Net
       end
     end
 
+    deprecate_constant :BodyTypeAttachment
+
     # Net::IMAP::BodyTypeMultipart represents body structures of messages and
     # message parts, when <tt>Content-Type</tt> is <tt>multipart/*</tt>.
     class BodyTypeMultipart < Struct.new(:media_type, :subtype,
                                          :parts,
                                          :param, :disposition, :language,
+                                         :location,
                                          :extension)
       include BodyStructure
 
@@ -1265,23 +1269,24 @@ module Net
       end
     end
 
-    # === WARNING:
+    # === Obsolete
+    # BodyTypeExtension is not used and will be removed in an upcoming release.
+    #
     # >>>
-    #   BodyTypeExtension is (incorrectly) used for <tt>message/*</tt> parts
+    #   BodyTypeExtension was (incorrectly) used for <tt>message/*</tt> parts
     #   (besides <tt>message/rfc822</tt>, which correctly uses BodyTypeMessage).
     #
-    # A future release will replace this class with:
-    # * BodyTypeMessage for <tt>message/rfc822</tt> and <tt>message/global</tt>
-    # * BodyTypeBasic for any other <tt>message/*</tt>
+    #   Net::IMAP now (correctly) parses all message types (other than
+    #   <tt>message/rfc822</tt> or <tt>message/global</tt>) as BodyTypeBasic.
     class BodyTypeExtension < Struct.new(:media_type, :subtype,
                                          :params, :content_id,
                                          :description, :encoding, :size)
-      include BodyStructure
-
       def multipart?
         return false
       end
     end
 
+    deprecate_constant :BodyTypeExtension
+
   end
 end

data/lib/net/imap/response_parser/parser_utils.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+module Net
+  class IMAP < Protocol
+    class ResponseParser
+      # basic utility methods for parsing.
+      #
+      # (internal API, subject to change)
+      module ParserUtils # :nodoc:
+
+        module Generator
+
+          LOOKAHEAD = "(@token ||= next_token)"
+          SHIFT_TOKEN = "(@token = nil)"
+
+          # we can skip lexer for single character matches, as a shortcut
+          def def_char_matchers(name, char, token)
+            match_name = name.match(/\A[A-Z]/) ? "#{name}!" : name
+            char = char.dump
+            class_eval <<~RUBY, __FILE__, __LINE__ + 1
+              # frozen_string_literal: true
+
+              # force use of #next_token; no string peeking
+              def lookahead_#{name}?
+                #{LOOKAHEAD}&.symbol == #{token}
+              end
+
+              # use token or string peek
+              def peek_#{name}?
+                @token ? @token.symbol == #{token} : @str[@pos] == #{char}
+              end
+
+              # like accept(token_symbols); returns token or nil
+              def #{name}?
+                if @token&.symbol == #{token}
+                  #{SHIFT_TOKEN}
+                  #{char}
+                elsif !@token && @str[@pos] == #{char}
+                  @pos += 1
+                  #{char}
+                end
+              end
+
+              # like match(token_symbols); returns token or raises parse_error
+              def #{match_name}
+                if @token&.symbol == #{token}
+                  #{SHIFT_TOKEN}
+                  #{char}
+                elsif !@token && @str[@pos] == #{char}
+                  @pos += 1
+                  #{char}
+                else
+                  parse_error("unexpected %s (expected %p)",
+                              @token&.symbol || @str[@pos].inspect, #{char})
+                end
+              end
+            RUBY
+          end
+
+          # TODO: move coersion to the token.value method?
+          def def_token_matchers(name, *token_symbols, coerce: nil, send: nil)
+            match_name = name.match(/\A[A-Z]/) ? "#{name}!" : name
+
+            if token_symbols.size == 1
+              token   = token_symbols.first
+              matcher = "token&.symbol == %p" % [token]
+              desc    = token
+            else
+              matcher = "%p.include? token&.symbol" % [token_symbols]
+              desc    = token_symbols.join(" or ")
+            end
+
+            value = "(token.value)"
+            value = coerce.to_s + value   if coerce
+            value = [value, send].join(".") if send
+
+            raise_parse_error = <<~RUBY
+              parse_error("unexpected %s (expected #{desc})", token&.symbol)
+            RUBY
+
+            class_eval <<~RUBY, __FILE__, __LINE__ + 1
+              # frozen_string_literal: true
+
+              # lookahead version of match, returning the value
+              def lookahead_#{name}!
+                token = #{LOOKAHEAD}
+                if #{matcher}
+                  #{value}
+                else
+                  #{raise_parse_error}
+                end
+              end
+
+              def #{name}?
+                token = #{LOOKAHEAD}
+                if #{matcher}
+                  #{SHIFT_TOKEN}
+                  #{value}
+                end
+              end
+
+              def #{match_name}
+                token = #{LOOKAHEAD}
+                if #{matcher}
+                  #{SHIFT_TOKEN}
+                  #{value}
+                else
+                  #{raise_parse_error}
+                end
+              end
+            RUBY
+
+          end
+
+        end
+
+        private
+
+        # TODO: after checking the lookahead, use a regexp for remaining chars.
+        # That way a loop isn't needed.
+        def combine_adjacent(*tokens)
+          result = "".b
+          while token = accept(*tokens)
+            result << token.value
+          end
+          if result.empty?
+            parse_error('unexpected token %s (expected %s)',
+                        lookahead.symbol, tokens.join(" or "))
+          end
+          result
+        end
+
+        def match(*args)
+          token = lookahead
+          unless args.include?(token.symbol)
+            parse_error('unexpected token %s (expected %s)',
+                        token.symbol.id2name,
+                        args.collect {|i| i.id2name}.join(" or "))
+          end
+          shift_token
+          token
+        end
+
+        # like match, but does not raise error on failure.
+        #
+        # returns and shifts token on successful match
+        # returns nil and leaves @token unshifted on no match
+        def accept(*args)
+          token = lookahead
+          if args.include?(token.symbol)
+            shift_token
+            token
+          end
+        end
+
+        # To be used conditionally:
+        #   assert_no_lookahead if Net::IMAP.debug
+        def assert_no_lookahead
+          @token.nil? or
+            parse_error("assertion failed: expected @token.nil?, actual %s: %p",
+                        @token.symbol, @token.value)
+        end
+
+        # like accept, without consuming the token
+        def lookahead?(*symbols)
+          @token if symbols.include?((@token ||= next_token)&.symbol)
+        end
+
+        def lookahead
+          @token ||= next_token
+        end
+
+        def peek_str?(str)
+          assert_no_lookahead if Net::IMAP.debug
+          @str[@pos, str.length] == str
+        end
+
+        def peek_re(re)
+          assert_no_lookahead if Net::IMAP.debug
+          re.match(@str, @pos)
+        end
+
+        def accept_re(re)
+          assert_no_lookahead if Net::IMAP.debug
+          re.match(@str, @pos) and @pos = $~.end(0)
+          $~
+        end
+
+        def match_re(re, name)
+          assert_no_lookahead if Net::IMAP.debug
+          if re.match(@str, @pos)
+            @pos = $~.end(0)
+            $~
+          else
+            parse_error("invalid #{name}")
+          end
+        end
+
+        def shift_token
+          @token = nil
+        end
+
+        def parse_error(fmt, *args)
+          msg = format(fmt, *args)
+          if IMAP.debug
+            local_path = File.dirname(__dir__)
+            tok = @token ? "%s: %p" % [@token.symbol, @token.value] : "nil"
+            warn "%s %s: %s"        % [self.class, __method__, msg]
+            warn "  tokenized : %s" % [@str[...@pos].dump]
+            warn "  remaining : %s" % [@str[@pos..].dump]
+            warn "  @lex_state: %s" % [@lex_state]
+            warn "  @pos      : %d" % [@pos]
+            warn "  @token    : %s" % [tok]
+            caller_locations(1..20).each_with_index do |cloc, idx|
+              next unless cloc.path&.start_with?(local_path)
+              warn "  caller[%2d]: %-30s (%s:%d)" % [
+                idx,
+                cloc.base_label,
+                File.basename(cloc.path, ".rb"),
+                cloc.lineno
+              ]
+            end
+          end
+          raise ResponseParseError, msg
+        end
+
+      end
+    end
+  end
+end