net-imap 0.3.4 → 0.4.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/pages.yml +46 -0
  3. data/.github/workflows/test.yml +12 -12
  4. data/Gemfile +1 -0
  5. data/README.md +15 -4
  6. data/Rakefile +0 -7
  7. data/benchmarks/generate_parser_benchmarks +52 -0
  8. data/benchmarks/parser.yml +578 -0
  9. data/benchmarks/stringprep.yml +1 -1
  10. data/lib/net/imap/authenticators.rb +26 -57
  11. data/lib/net/imap/command_data.rb +13 -6
  12. data/lib/net/imap/data_encoding.rb +3 -3
  13. data/lib/net/imap/deprecated_client_options.rb +139 -0
  14. data/lib/net/imap/response_data.rb +46 -41
  15. data/lib/net/imap/response_parser/parser_utils.rb +230 -0
  16. data/lib/net/imap/response_parser.rb +665 -627
  17. data/lib/net/imap/sasl/anonymous_authenticator.rb +68 -0
  18. data/lib/net/imap/sasl/authentication_exchange.rb +107 -0
  19. data/lib/net/imap/sasl/authenticators.rb +118 -0
  20. data/lib/net/imap/sasl/client_adapter.rb +72 -0
  21. data/lib/net/imap/{authenticators/cram_md5.rb → sasl/cram_md5_authenticator.rb} +15 -9
  22. data/lib/net/imap/sasl/digest_md5_authenticator.rb +168 -0
  23. data/lib/net/imap/sasl/external_authenticator.rb +62 -0
  24. data/lib/net/imap/sasl/gs2_header.rb +80 -0
  25. data/lib/net/imap/{authenticators/login.rb → sasl/login_authenticator.rb} +19 -14
  26. data/lib/net/imap/sasl/oauthbearer_authenticator.rb +164 -0
  27. data/lib/net/imap/sasl/plain_authenticator.rb +93 -0
  28. data/lib/net/imap/sasl/protocol_adapters.rb +45 -0
  29. data/lib/net/imap/sasl/scram_algorithm.rb +58 -0
  30. data/lib/net/imap/sasl/scram_authenticator.rb +278 -0
  31. data/lib/net/imap/sasl/stringprep.rb +6 -66
  32. data/lib/net/imap/sasl/xoauth2_authenticator.rb +88 -0
  33. data/lib/net/imap/sasl.rb +144 -43
  34. data/lib/net/imap/sasl_adapter.rb +21 -0
  35. data/lib/net/imap/stringprep/nameprep.rb +70 -0
  36. data/lib/net/imap/stringprep/saslprep.rb +69 -0
  37. data/lib/net/imap/stringprep/saslprep_tables.rb +96 -0
  38. data/lib/net/imap/stringprep/tables.rb +146 -0
  39. data/lib/net/imap/stringprep/trace.rb +85 -0
  40. data/lib/net/imap/stringprep.rb +159 -0
  41. data/lib/net/imap.rb +976 -590
  42. data/net-imap.gemspec +2 -2
  43. data/rakelib/saslprep.rake +4 -4
  44. data/rakelib/string_prep_tables_generator.rb +82 -60
  45. metadata +31 -12
  46. data/lib/net/imap/authenticators/digest_md5.rb +0 -115
  47. data/lib/net/imap/authenticators/plain.rb +0 -41
  48. data/lib/net/imap/authenticators/xoauth2.rb +0 -20
  49. data/lib/net/imap/sasl/saslprep.rb +0 -55
  50. data/lib/net/imap/sasl/saslprep_tables.rb +0 -98
  51. data/lib/net/imap/sasl/stringprep_tables.rb +0 -153
@@ -1,68 +1,37 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # Registry for SASL authenticators used by Net::IMAP.
3
+ # Backward compatible delegators from Net::IMAP to Net::IMAP::SASL.
4
4
  module Net::IMAP::Authenticators
5
5
 
6
- # Adds an authenticator for Net::IMAP#authenticate to use. +mechanism+ is the
7
- # {SASL mechanism}[https://www.iana.org/assignments/sasl-mechanisms/sasl-mechanisms.xhtml]
8
- # implemented by +authenticator+ (for instance, <tt>"PLAIN"</tt>).
9
- #
10
- # The +authenticator+ must respond to +#new+ (or #call), receiving the
11
- # authenticator configuration and return a configured authentication session.
12
- # The authenticator session must respond to +#process+, receiving the server's
13
- # challenge and returning the client's response.
14
- #
15
- # See PlainAuthenticator, XOauth2Authenticator, and DigestMD5Authenticator for
16
- # examples.
17
- def add_authenticator(auth_type, authenticator)
18
- authenticators[auth_type] = authenticator
6
+ # Deprecated. Use Net::IMAP::SASL.add_authenticator instead.
7
+ def add_authenticator(...)
8
+ warn(
9
+ "%s.%s is deprecated. Use %s.%s instead." % [
10
+ Net::IMAP, __method__, Net::IMAP::SASL, __method__
11
+ ],
12
+ uplevel: 1
13
+ )
14
+ Net::IMAP::SASL.add_authenticator(...)
19
15
  end
20
16
 
21
- # :call-seq:
22
- # authenticator(mechanism, ...) -> authenticator
23
- # authenticator(mech, *creds, **props) {|prop, auth| val } -> authenticator
24
- # authenticator(mechanism, authnid, creds, authzid=nil) -> authenticator
25
- # authenticator(mechanism, **properties) -> authenticator
26
- # authenticator(mechanism) {|propname, authctx| value } -> authenticator
27
- #
28
- # Builds a new authentication session context for +mechanism+.
29
- #
30
- # [Note]
31
- # This method is intended for internal use by connection protocol code only.
32
- # Protocol client users should see refer to their client's documentation,
33
- # e.g. Net::IMAP#authenticate for Net::IMAP.
34
- #
35
- # The call signatures documented for this method are recommendations for
36
- # authenticator implementors. All arguments (other than +mechanism+) are
37
- # forwarded to the registered authenticator's +#new+ (or +#call+) method, and
38
- # each authenticator must document its own arguments.
39
- #
40
- # The returned object represents a single authentication exchange and <em>must
41
- # not</em> be reused for multiple authentication attempts.
42
- def authenticator(mechanism, *authargs, **properties, &callback)
43
- authenticator = authenticators.fetch(mechanism.upcase) do
44
- raise ArgumentError, 'unknown auth type - "%s"' % mechanism
45
- end
46
- if authenticator.respond_to?(:new)
47
- authenticator.new(*authargs, **properties, &callback)
48
- else
49
- authenticator.call(*authargs, **properties, &callback)
50
- end
51
- end
52
-
53
- private
54
-
55
- def authenticators
56
- @authenticators ||= {}
17
+ # Deprecated. Use Net::IMAP::SASL.authenticator instead.
18
+ def authenticator(...)
19
+ warn(
20
+ "%s.%s is deprecated. Use %s.%s instead." % [
21
+ Net::IMAP, __method__, Net::IMAP::SASL, __method__
22
+ ],
23
+ uplevel: 1
24
+ )
25
+ Net::IMAP::SASL.authenticator(...)
57
26
  end
58
27
 
28
+ Net::IMAP.extend self
59
29
  end
60
30
 
61
- Net::IMAP.extend Net::IMAP::Authenticators
31
+ class Net::IMAP
32
+ PlainAuthenticator = SASL::PlainAuthenticator # :nodoc:
33
+ deprecate_constant :PlainAuthenticator
62
34
 
63
- require_relative "authenticators/plain"
64
-
65
- require_relative "authenticators/login"
66
- require_relative "authenticators/cram_md5"
67
- require_relative "authenticators/digest_md5"
68
- require_relative "authenticators/xoauth2"
35
+ XOauth2Authenticator = SASL::XOAuth2Authenticator # :nodoc:
36
+ deprecate_constant :XOauth2Authenticator
37
+ end
@@ -52,13 +52,20 @@ module Net
52
52
  end
53
53
 
54
54
  def send_string_data(str, tag = nil)
55
- case str
56
- when ""
55
+ if str.empty?
57
56
  put_string('""')
58
- when /[\x80-\xff\r\n]/n
59
- # literal
57
+ elsif str.match?(/[\r\n]/n)
58
+ # literal, because multiline
60
59
  send_literal(str, tag)
61
- when /[(){ \x00-\x1f\x7f%*"\\]/n
60
+ elsif !str.ascii_only?
61
+ if @utf8_strings
62
+ # quoted string
63
+ send_quoted_string(str)
64
+ else
65
+ # literal, because of non-ASCII bytes
66
+ send_literal(str, tag)
67
+ end
68
+ elsif str.match?(/[(){ \x00-\x1f\x7f%*"\\]/n)
62
69
  # quoted string
63
70
  send_quoted_string(str)
64
71
  else
@@ -67,7 +74,7 @@ module Net
67
74
  end
68
75
 
69
76
  def send_quoted_string(str)
70
- put_string('"' + str.gsub(/["\\]/n, "\\\\\\&") + '"')
77
+ put_string('"' + str.gsub(/["\\]/, "\\\\\\&") + '"')
71
78
  end
72
79
 
73
80
  def send_literal(str, tag = nil)
@@ -54,9 +54,9 @@ module Net
54
54
  # Net::IMAP does _not_ automatically encode and decode
55
55
  # mailbox names to and from UTF-7.
56
56
  def self.decode_utf7(s)
57
- return s.gsub(/&([^-]+)?-/n) {
58
- if $1
59
- ($1.tr(",", "/") + "===").unpack1("m").encode(Encoding::UTF_8, Encoding::UTF_16BE)
57
+ return s.gsub(/&([A-Za-z0-9+,]+)?-/n) {
58
+ if base64 = $1
59
+ (base64.tr(",", "/") + "===").unpack1("m").encode(Encoding::UTF_8, Encoding::UTF_16BE)
60
60
  else
61
61
  "&"
62
62
  end
@@ -0,0 +1,139 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Net
4
+ class IMAP < Protocol
5
+
6
+ # This module handles deprecated arguments to various Net::IMAP methods.
7
+ module DeprecatedClientOptions
8
+
9
+ # :call-seq:
10
+ # Net::IMAP.new(host, **options) # standard keyword options
11
+ # Net::IMAP.new(host, options) # obsolete hash options
12
+ # Net::IMAP.new(host, port) # obsolete port argument
13
+ # Net::IMAP.new(host, port, usessl, certs = nil, verify = true) # deprecated SSL arguments
14
+ #
15
+ # Translates Net::IMAP.new arguments for backward compatibility.
16
+ #
17
+ # ==== Obsolete arguments
18
+ #
19
+ # Using obsolete arguments does not a warning. Obsolete arguments will be
20
+ # deprecated by a future release.
21
+ #
22
+ # If a second positional argument is given and it is a hash (or is
23
+ # convertable via +#to_hash+), it is converted to keyword arguments.
24
+ #
25
+ # # Obsolete:
26
+ # Net::IMAP.new("imap.example.com", options_hash)
27
+ # # Use instead:
28
+ # Net::IMAP.new("imap.example.com", **options_hash)
29
+ #
30
+ # If a second positional argument is given and it is not a hash, it is
31
+ # converted to the +port+ keyword argument.
32
+ # # Obsolete:
33
+ # Net::IMAP.new("imap.example.com", 114433)
34
+ # # Use instead:
35
+ # Net::IMAP.new("imap.example.com", port: 114433)
36
+ #
37
+ # ==== Deprecated arguments
38
+ #
39
+ # Using deprecated arguments prints a warning. Convert to keyword
40
+ # arguments to avoid the warning. Deprecated arguments will be removed in
41
+ # a future release.
42
+ #
43
+ # If +usessl+ is false, +certs+, and +verify+ are ignored. When it true,
44
+ # all three arguments are converted to the +ssl+ keyword argument.
45
+ # Without +certs+ or +verify+, it is converted to <tt>ssl: true</tt>.
46
+ # # DEPRECATED:
47
+ # Net::IMAP.new("imap.example.com", nil, true) # => prints a warning
48
+ # # Use instead:
49
+ # Net::IMAP.new("imap.example.com", ssl: true)
50
+ #
51
+ # When +certs+ is a path to a directory, it is converted to <tt>ca_path:
52
+ # certs</tt>.
53
+ # # DEPRECATED:
54
+ # Net::IMAP.new("imap.example.com", nil, true, "/path/to/certs") # => prints a warning
55
+ # # Use instead:
56
+ # Net::IMAP.new("imap.example.com", ssl: {ca_path: "/path/to/certs"})
57
+ #
58
+ # When +certs+ is a path to a file, it is converted to <tt>ca_file:
59
+ # certs</tt>.
60
+ # # DEPRECATED:
61
+ # Net::IMAP.new("imap.example.com", nil, true, "/path/to/cert.pem") # => prints a warning
62
+ # # Use instead:
63
+ # Net::IMAP.new("imap.example.com", ssl: {ca_file: "/path/to/cert.pem"})
64
+ #
65
+ # When +verify+ is +false+, it is converted to <tt>verify_mode:
66
+ # OpenSSL::SSL::VERIFY_NONE</tt>.
67
+ # # DEPRECATED:
68
+ # Net::IMAP.new("imap.example.com", nil, true, nil, false) # => prints a warning
69
+ # # Use instead:
70
+ # Net::IMAP.new("imap.example.com", ssl: {verify_mode: OpenSSL::SSL::VERIFY_NONE})
71
+ #
72
+ def initialize(host, port_or_options = nil, *deprecated, **options)
73
+ if port_or_options.nil? && deprecated.empty?
74
+ super host, **options
75
+ elsif options.any?
76
+ # Net::IMAP.new(host, *__invalid__, **options)
77
+ raise ArgumentError, "Do not combine deprecated and keyword arguments"
78
+ elsif port_or_options.respond_to?(:to_hash) and deprecated.any?
79
+ # Net::IMAP.new(host, options, *__invalid__)
80
+ raise ArgumentError, "Do not use deprecated SSL params with options hash"
81
+ elsif port_or_options.respond_to?(:to_hash)
82
+ super host, **Hash.try_convert(port_or_options)
83
+ elsif deprecated.empty?
84
+ super host, port: port_or_options
85
+ elsif deprecated.shift
86
+ warn "DEPRECATED: Call Net::IMAP.new with keyword options", uplevel: 1
87
+ super host, port: port_or_options, ssl: create_ssl_params(*deprecated)
88
+ else
89
+ warn "DEPRECATED: Call Net::IMAP.new with keyword options", uplevel: 1
90
+ super host, port: port_or_options, ssl: false
91
+ end
92
+ end
93
+
94
+ # :call-seq:
95
+ # starttls(**options) # standard
96
+ # starttls(options = {}) # obsolete
97
+ # starttls(certs = nil, verify = true) # deprecated
98
+ #
99
+ # Translates Net::IMAP#starttls arguments for backward compatibility.
100
+ #
101
+ # Support for +certs+ and +verify+ will be dropped in a future release.
102
+ #
103
+ # See ::new for interpretation of +certs+ and +verify+.
104
+ def starttls(*deprecated, **options)
105
+ if deprecated.empty?
106
+ super(**options)
107
+ elsif options.any?
108
+ # starttls(*__invalid__, **options)
109
+ raise ArgumentError, "Do not combine deprecated and keyword options"
110
+ elsif deprecated.first.respond_to?(:to_hash) && deprecated.length > 1
111
+ # starttls(*__invalid__, **options)
112
+ raise ArgumentError, "Do not use deprecated verify param with options hash"
113
+ elsif deprecated.first.respond_to?(:to_hash)
114
+ super(**Hash.try_convert(deprecated.first))
115
+ else
116
+ warn "DEPRECATED: Call Net::IMAP#starttls with keyword options", uplevel: 1
117
+ super(**create_ssl_params(*deprecated))
118
+ end
119
+ end
120
+
121
+ private
122
+
123
+ def create_ssl_params(certs = nil, verify = true)
124
+ params = {}
125
+ if certs
126
+ if File.file?(certs)
127
+ params[:ca_file] = certs
128
+ elsif File.directory?(certs)
129
+ params[:ca_path] = certs
130
+ end
131
+ end
132
+ params[:verify_mode] =
133
+ verify ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
134
+ params
135
+ end
136
+
137
+ end
138
+ end
139
+ end
@@ -891,13 +891,6 @@ module Net
891
891
  # should use BodyTypeBasic.
892
892
  # BodyTypeMultipart:: for <tt>multipart/*</tt> parts
893
893
  #
894
- # ==== Deprecated BodyStructure classes
895
- # The following classes represent invalid server responses or parser bugs:
896
- # BodyTypeExtension:: parser bug: used for <tt>message/*</tt> where
897
- # BodyTypeBasic should have been used.
898
- # BodyTypeAttachment:: server bug: some servers sometimes return the
899
- # "Content-Disposition: attachment" data where the
900
- # entire body structure for a message part is expected.
901
894
  module BodyStructure
902
895
  end
903
896
 
@@ -914,6 +907,7 @@ module Net
914
907
  :param, :content_id,
915
908
  :description, :encoding, :size,
916
909
  :md5, :disposition, :language,
910
+ :location,
917
911
  :extension)
918
912
  include BodyStructure
919
913
 
@@ -1049,6 +1043,7 @@ module Net
1049
1043
  :description, :encoding, :size,
1050
1044
  :lines,
1051
1045
  :md5, :disposition, :language,
1046
+ :location,
1052
1047
  :extension)
1053
1048
  include BodyStructure
1054
1049
 
@@ -1094,6 +1089,7 @@ module Net
1094
1089
  :description, :encoding, :size,
1095
1090
  :envelope, :body, :lines,
1096
1091
  :md5, :disposition, :language,
1092
+ :location,
1097
1093
  :extension)
1098
1094
  include BodyStructure
1099
1095
 
@@ -1126,36 +1122,41 @@ module Net
1126
1122
  end
1127
1123
  end
1128
1124
 
1129
- # === WARNING
1130
- # BodyTypeAttachment represents a <tt>body-fld-dsp</tt> that is
1131
- # incorrectly in a position where the IMAP4rev1 grammar expects a nested
1132
- # +body+ structure.
1125
+ # BodyTypeAttachment is not used and will be removed in an upcoming release.
1133
1126
  #
1134
- # >>>
1135
- # \IMAP body structures are parenthesized lists and assign their fields
1136
- # positionally, so missing fields change the intepretation of all
1137
- # following fields. Buggy \IMAP servers sometimes leave fields missing
1138
- # rather than empty, which inevitably confuses parsers.
1139
- # BodyTypeAttachment was an attempt to parse a common type of buggy body
1140
- # structure without crashing.
1141
- #
1142
- # Currently, when Net::IMAP::ResponseParser sees "attachment" as the first
1143
- # entry in a <tt>body-type-1part</tt>, which is where the MIME type should
1144
- # be, it uses BodyTypeAttachment to capture the rest. "attachment" is not
1145
- # a valid MIME type, but _is_ a common <tt>Content-Disposition</tt>. What
1146
- # might have happened was that buggy server could not parse the message
1147
- # (which might have been incorrectly formatted) and output a
1148
- # <tt>body-type-dsp</tt> where a Net::IMAP::ResponseParser expected to see
1149
- # a +body+.
1150
- #
1151
- # A future release will replace this, probably with a ContentDisposition
1152
- # nested inside another body structure object, maybe BodyTypeBasic, or
1153
- # perhaps a new body structure class that represents any unparsable body
1154
- # structure.
1127
+ # === Bug Analysis
1128
+ #
1129
+ # \IMAP body structures are parenthesized lists and assign their fields
1130
+ # positionally, so missing fields change the intepretation of all
1131
+ # following fields. Additionally, different body types have a different
1132
+ # number of required fields, followed by optional "extension" fields.
1133
+ #
1134
+ # BodyTypeAttachment was previously returned when a "message/rfc822" part,
1135
+ # which should be sent as <tt>body-type-msg</tt> with ten required fields,
1136
+ # was actually sent as a <tt>body-type-basic</tt> with _seven_ required
1137
+ # fields.
1138
+ #
1139
+ # basic => type, subtype, param, id, desc, enc, octets, md5=nil, dsp=nil, lang=nil, loc=nil, *ext
1140
+ # msg => type, subtype, param, id, desc, enc, octets, envelope, body, lines, md5=nil, ...
1141
+ #
1142
+ # Normally, +envelope+ and +md5+ are incompatible, but Net::IMAP leniently
1143
+ # allowed buggy servers to send +NIL+ for +envelope+. As a result, when a
1144
+ # server sent a <tt>message/rfc822</tt> part with +NIL+ for +md5+ and a
1145
+ # non-<tt>NIL</tt> +dsp+, Net::IMAP mis-interpreted the
1146
+ # <tt>Content-Disposition</tt> as if it were a strange body type. In all
1147
+ # reported cases, the <tt>Content-Disposition</tt> was "attachment", so
1148
+ # BodyTypeAttachment was created as the workaround.
1149
+ #
1150
+ # === Current behavior
1151
+ #
1152
+ # When interpreted strictly, +envelope+ and +md5+ are incompatible. So the
1153
+ # current parsing algorithm peeks ahead after it has recieved the seventh
1154
+ # body field. If the next token is not the start of an +envelope+, we assume
1155
+ # the server has incorrectly sent us a <tt>body-type-basic</tt> and return
1156
+ # BodyTypeBasic. As a result, what was previously BodyTypeMessage#body =>
1157
+ # BodyTypeAttachment is now BodyTypeBasic#disposition => ContentDisposition.
1155
1158
  #
1156
1159
  class BodyTypeAttachment < Struct.new(:dsp_type, :_unused_, :param)
1157
- include BodyStructure
1158
-
1159
1160
  # *invalid for BodyTypeAttachment*
1160
1161
  def media_type
1161
1162
  warn(<<~WARN, uplevel: 1)
@@ -1190,11 +1191,14 @@ module Net
1190
1191
  end
1191
1192
  end
1192
1193
 
1194
+ deprecate_constant :BodyTypeAttachment
1195
+
1193
1196
  # Net::IMAP::BodyTypeMultipart represents body structures of messages and
1194
1197
  # message parts, when <tt>Content-Type</tt> is <tt>multipart/*</tt>.
1195
1198
  class BodyTypeMultipart < Struct.new(:media_type, :subtype,
1196
1199
  :parts,
1197
1200
  :param, :disposition, :language,
1201
+ :location,
1198
1202
  :extension)
1199
1203
  include BodyStructure
1200
1204
 
@@ -1265,23 +1269,24 @@ module Net
1265
1269
  end
1266
1270
  end
1267
1271
 
1268
- # === WARNING:
1272
+ # === Obsolete
1273
+ # BodyTypeExtension is not used and will be removed in an upcoming release.
1274
+ #
1269
1275
  # >>>
1270
- # BodyTypeExtension is (incorrectly) used for <tt>message/*</tt> parts
1276
+ # BodyTypeExtension was (incorrectly) used for <tt>message/*</tt> parts
1271
1277
  # (besides <tt>message/rfc822</tt>, which correctly uses BodyTypeMessage).
1272
1278
  #
1273
- # A future release will replace this class with:
1274
- # * BodyTypeMessage for <tt>message/rfc822</tt> and <tt>message/global</tt>
1275
- # * BodyTypeBasic for any other <tt>message/*</tt>
1279
+ # Net::IMAP now (correctly) parses all message types (other than
1280
+ # <tt>message/rfc822</tt> or <tt>message/global</tt>) as BodyTypeBasic.
1276
1281
  class BodyTypeExtension < Struct.new(:media_type, :subtype,
1277
1282
  :params, :content_id,
1278
1283
  :description, :encoding, :size)
1279
- include BodyStructure
1280
-
1281
1284
  def multipart?
1282
1285
  return false
1283
1286
  end
1284
1287
  end
1285
1288
 
1289
+ deprecate_constant :BodyTypeExtension
1290
+
1286
1291
  end
1287
1292
  end
@@ -0,0 +1,230 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Net
4
+ class IMAP < Protocol
5
+ class ResponseParser
6
+ # basic utility methods for parsing.
7
+ #
8
+ # (internal API, subject to change)
9
+ module ParserUtils # :nodoc:
10
+
11
+ module Generator
12
+
13
+ LOOKAHEAD = "(@token ||= next_token)"
14
+ SHIFT_TOKEN = "(@token = nil)"
15
+
16
+ # we can skip lexer for single character matches, as a shortcut
17
+ def def_char_matchers(name, char, token)
18
+ match_name = name.match(/\A[A-Z]/) ? "#{name}!" : name
19
+ char = char.dump
20
+ class_eval <<~RUBY, __FILE__, __LINE__ + 1
21
+ # frozen_string_literal: true
22
+
23
+ # force use of #next_token; no string peeking
24
+ def lookahead_#{name}?
25
+ #{LOOKAHEAD}&.symbol == #{token}
26
+ end
27
+
28
+ # use token or string peek
29
+ def peek_#{name}?
30
+ @token ? @token.symbol == #{token} : @str[@pos] == #{char}
31
+ end
32
+
33
+ # like accept(token_symbols); returns token or nil
34
+ def #{name}?
35
+ if @token&.symbol == #{token}
36
+ #{SHIFT_TOKEN}
37
+ #{char}
38
+ elsif !@token && @str[@pos] == #{char}
39
+ @pos += 1
40
+ #{char}
41
+ end
42
+ end
43
+
44
+ # like match(token_symbols); returns token or raises parse_error
45
+ def #{match_name}
46
+ if @token&.symbol == #{token}
47
+ #{SHIFT_TOKEN}
48
+ #{char}
49
+ elsif !@token && @str[@pos] == #{char}
50
+ @pos += 1
51
+ #{char}
52
+ else
53
+ parse_error("unexpected %s (expected %p)",
54
+ @token&.symbol || @str[@pos].inspect, #{char})
55
+ end
56
+ end
57
+ RUBY
58
+ end
59
+
60
+ # TODO: move coersion to the token.value method?
61
+ def def_token_matchers(name, *token_symbols, coerce: nil, send: nil)
62
+ match_name = name.match(/\A[A-Z]/) ? "#{name}!" : name
63
+
64
+ if token_symbols.size == 1
65
+ token = token_symbols.first
66
+ matcher = "token&.symbol == %p" % [token]
67
+ desc = token
68
+ else
69
+ matcher = "%p.include? token&.symbol" % [token_symbols]
70
+ desc = token_symbols.join(" or ")
71
+ end
72
+
73
+ value = "(token.value)"
74
+ value = coerce.to_s + value if coerce
75
+ value = [value, send].join(".") if send
76
+
77
+ raise_parse_error = <<~RUBY
78
+ parse_error("unexpected %s (expected #{desc})", token&.symbol)
79
+ RUBY
80
+
81
+ class_eval <<~RUBY, __FILE__, __LINE__ + 1
82
+ # frozen_string_literal: true
83
+
84
+ # lookahead version of match, returning the value
85
+ def lookahead_#{name}!
86
+ token = #{LOOKAHEAD}
87
+ if #{matcher}
88
+ #{value}
89
+ else
90
+ #{raise_parse_error}
91
+ end
92
+ end
93
+
94
+ def #{name}?
95
+ token = #{LOOKAHEAD}
96
+ if #{matcher}
97
+ #{SHIFT_TOKEN}
98
+ #{value}
99
+ end
100
+ end
101
+
102
+ def #{match_name}
103
+ token = #{LOOKAHEAD}
104
+ if #{matcher}
105
+ #{SHIFT_TOKEN}
106
+ #{value}
107
+ else
108
+ #{raise_parse_error}
109
+ end
110
+ end
111
+ RUBY
112
+
113
+ end
114
+
115
+ end
116
+
117
+ private
118
+
119
+ # TODO: after checking the lookahead, use a regexp for remaining chars.
120
+ # That way a loop isn't needed.
121
+ def combine_adjacent(*tokens)
122
+ result = "".b
123
+ while token = accept(*tokens)
124
+ result << token.value
125
+ end
126
+ if result.empty?
127
+ parse_error('unexpected token %s (expected %s)',
128
+ lookahead.symbol, tokens.join(" or "))
129
+ end
130
+ result
131
+ end
132
+
133
+ def match(*args)
134
+ token = lookahead
135
+ unless args.include?(token.symbol)
136
+ parse_error('unexpected token %s (expected %s)',
137
+ token.symbol.id2name,
138
+ args.collect {|i| i.id2name}.join(" or "))
139
+ end
140
+ shift_token
141
+ token
142
+ end
143
+
144
+ # like match, but does not raise error on failure.
145
+ #
146
+ # returns and shifts token on successful match
147
+ # returns nil and leaves @token unshifted on no match
148
+ def accept(*args)
149
+ token = lookahead
150
+ if args.include?(token.symbol)
151
+ shift_token
152
+ token
153
+ end
154
+ end
155
+
156
+ # To be used conditionally:
157
+ # assert_no_lookahead if Net::IMAP.debug
158
+ def assert_no_lookahead
159
+ @token.nil? or
160
+ parse_error("assertion failed: expected @token.nil?, actual %s: %p",
161
+ @token.symbol, @token.value)
162
+ end
163
+
164
+ # like accept, without consuming the token
165
+ def lookahead?(*symbols)
166
+ @token if symbols.include?((@token ||= next_token)&.symbol)
167
+ end
168
+
169
+ def lookahead
170
+ @token ||= next_token
171
+ end
172
+
173
+ def peek_str?(str)
174
+ assert_no_lookahead if Net::IMAP.debug
175
+ @str[@pos, str.length] == str
176
+ end
177
+
178
+ def peek_re(re)
179
+ assert_no_lookahead if Net::IMAP.debug
180
+ re.match(@str, @pos)
181
+ end
182
+
183
+ def accept_re(re)
184
+ assert_no_lookahead if Net::IMAP.debug
185
+ re.match(@str, @pos) and @pos = $~.end(0)
186
+ $~
187
+ end
188
+
189
+ def match_re(re, name)
190
+ assert_no_lookahead if Net::IMAP.debug
191
+ if re.match(@str, @pos)
192
+ @pos = $~.end(0)
193
+ $~
194
+ else
195
+ parse_error("invalid #{name}")
196
+ end
197
+ end
198
+
199
+ def shift_token
200
+ @token = nil
201
+ end
202
+
203
+ def parse_error(fmt, *args)
204
+ msg = format(fmt, *args)
205
+ if IMAP.debug
206
+ local_path = File.dirname(__dir__)
207
+ tok = @token ? "%s: %p" % [@token.symbol, @token.value] : "nil"
208
+ warn "%s %s: %s" % [self.class, __method__, msg]
209
+ warn " tokenized : %s" % [@str[...@pos].dump]
210
+ warn " remaining : %s" % [@str[@pos..].dump]
211
+ warn " @lex_state: %s" % [@lex_state]
212
+ warn " @pos : %d" % [@pos]
213
+ warn " @token : %s" % [tok]
214
+ caller_locations(1..20).each_with_index do |cloc, idx|
215
+ next unless cloc.path&.start_with?(local_path)
216
+ warn " caller[%2d]: %-30s (%s:%d)" % [
217
+ idx,
218
+ cloc.base_label,
219
+ File.basename(cloc.path, ".rb"),
220
+ cloc.lineno
221
+ ]
222
+ end
223
+ end
224
+ raise ResponseParseError, msg
225
+ end
226
+
227
+ end
228
+ end
229
+ end
230
+ end