net-imap 0.3.4 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of net-imap might be problematic. Click here for more details.

Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/pages.yml +46 -0
  3. data/.github/workflows/test.yml +12 -12
  4. data/Gemfile +1 -0
  5. data/README.md +15 -4
  6. data/Rakefile +0 -7
  7. data/benchmarks/generate_parser_benchmarks +52 -0
  8. data/benchmarks/parser.yml +578 -0
  9. data/benchmarks/stringprep.yml +1 -1
  10. data/lib/net/imap/authenticators.rb +26 -57
  11. data/lib/net/imap/command_data.rb +13 -6
  12. data/lib/net/imap/data_encoding.rb +3 -3
  13. data/lib/net/imap/deprecated_client_options.rb +139 -0
  14. data/lib/net/imap/response_data.rb +46 -41
  15. data/lib/net/imap/response_parser/parser_utils.rb +230 -0
  16. data/lib/net/imap/response_parser.rb +665 -627
  17. data/lib/net/imap/sasl/anonymous_authenticator.rb +68 -0
  18. data/lib/net/imap/sasl/authentication_exchange.rb +107 -0
  19. data/lib/net/imap/sasl/authenticators.rb +118 -0
  20. data/lib/net/imap/sasl/client_adapter.rb +72 -0
  21. data/lib/net/imap/{authenticators/cram_md5.rb → sasl/cram_md5_authenticator.rb} +15 -9
  22. data/lib/net/imap/sasl/digest_md5_authenticator.rb +168 -0
  23. data/lib/net/imap/sasl/external_authenticator.rb +62 -0
  24. data/lib/net/imap/sasl/gs2_header.rb +80 -0
  25. data/lib/net/imap/{authenticators/login.rb → sasl/login_authenticator.rb} +19 -14
  26. data/lib/net/imap/sasl/oauthbearer_authenticator.rb +164 -0
  27. data/lib/net/imap/sasl/plain_authenticator.rb +93 -0
  28. data/lib/net/imap/sasl/protocol_adapters.rb +45 -0
  29. data/lib/net/imap/sasl/scram_algorithm.rb +58 -0
  30. data/lib/net/imap/sasl/scram_authenticator.rb +278 -0
  31. data/lib/net/imap/sasl/stringprep.rb +6 -66
  32. data/lib/net/imap/sasl/xoauth2_authenticator.rb +88 -0
  33. data/lib/net/imap/sasl.rb +144 -43
  34. data/lib/net/imap/sasl_adapter.rb +21 -0
  35. data/lib/net/imap/stringprep/nameprep.rb +70 -0
  36. data/lib/net/imap/stringprep/saslprep.rb +69 -0
  37. data/lib/net/imap/stringprep/saslprep_tables.rb +96 -0
  38. data/lib/net/imap/stringprep/tables.rb +146 -0
  39. data/lib/net/imap/stringprep/trace.rb +85 -0
  40. data/lib/net/imap/stringprep.rb +159 -0
  41. data/lib/net/imap.rb +976 -590
  42. data/net-imap.gemspec +2 -2
  43. data/rakelib/saslprep.rake +4 -4
  44. data/rakelib/string_prep_tables_generator.rb +82 -60
  45. metadata +31 -12
  46. data/lib/net/imap/authenticators/digest_md5.rb +0 -115
  47. data/lib/net/imap/authenticators/plain.rb +0 -41
  48. data/lib/net/imap/authenticators/xoauth2.rb +0 -20
  49. data/lib/net/imap/sasl/saslprep.rb +0 -55
  50. data/lib/net/imap/sasl/saslprep_tables.rb +0 -98
  51. data/lib/net/imap/sasl/stringprep_tables.rb +0 -153
@@ -1,68 +1,37 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # Registry for SASL authenticators used by Net::IMAP.
3
+ # Backward compatible delegators from Net::IMAP to Net::IMAP::SASL.
4
4
  module Net::IMAP::Authenticators
5
5
 
6
- # Adds an authenticator for Net::IMAP#authenticate to use. +mechanism+ is the
7
- # {SASL mechanism}[https://www.iana.org/assignments/sasl-mechanisms/sasl-mechanisms.xhtml]
8
- # implemented by +authenticator+ (for instance, <tt>"PLAIN"</tt>).
9
- #
10
- # The +authenticator+ must respond to +#new+ (or #call), receiving the
11
- # authenticator configuration and return a configured authentication session.
12
- # The authenticator session must respond to +#process+, receiving the server's
13
- # challenge and returning the client's response.
14
- #
15
- # See PlainAuthenticator, XOauth2Authenticator, and DigestMD5Authenticator for
16
- # examples.
17
- def add_authenticator(auth_type, authenticator)
18
- authenticators[auth_type] = authenticator
6
+ # Deprecated. Use Net::IMAP::SASL.add_authenticator instead.
7
+ def add_authenticator(...)
8
+ warn(
9
+ "%s.%s is deprecated. Use %s.%s instead." % [
10
+ Net::IMAP, __method__, Net::IMAP::SASL, __method__
11
+ ],
12
+ uplevel: 1
13
+ )
14
+ Net::IMAP::SASL.add_authenticator(...)
19
15
  end
20
16
 
21
- # :call-seq:
22
- # authenticator(mechanism, ...) -> authenticator
23
- # authenticator(mech, *creds, **props) {|prop, auth| val } -> authenticator
24
- # authenticator(mechanism, authnid, creds, authzid=nil) -> authenticator
25
- # authenticator(mechanism, **properties) -> authenticator
26
- # authenticator(mechanism) {|propname, authctx| value } -> authenticator
27
- #
28
- # Builds a new authentication session context for +mechanism+.
29
- #
30
- # [Note]
31
- # This method is intended for internal use by connection protocol code only.
32
- # Protocol client users should see refer to their client's documentation,
33
- # e.g. Net::IMAP#authenticate for Net::IMAP.
34
- #
35
- # The call signatures documented for this method are recommendations for
36
- # authenticator implementors. All arguments (other than +mechanism+) are
37
- # forwarded to the registered authenticator's +#new+ (or +#call+) method, and
38
- # each authenticator must document its own arguments.
39
- #
40
- # The returned object represents a single authentication exchange and <em>must
41
- # not</em> be reused for multiple authentication attempts.
42
- def authenticator(mechanism, *authargs, **properties, &callback)
43
- authenticator = authenticators.fetch(mechanism.upcase) do
44
- raise ArgumentError, 'unknown auth type - "%s"' % mechanism
45
- end
46
- if authenticator.respond_to?(:new)
47
- authenticator.new(*authargs, **properties, &callback)
48
- else
49
- authenticator.call(*authargs, **properties, &callback)
50
- end
51
- end
52
-
53
- private
54
-
55
- def authenticators
56
- @authenticators ||= {}
17
+ # Deprecated. Use Net::IMAP::SASL.authenticator instead.
18
+ def authenticator(...)
19
+ warn(
20
+ "%s.%s is deprecated. Use %s.%s instead." % [
21
+ Net::IMAP, __method__, Net::IMAP::SASL, __method__
22
+ ],
23
+ uplevel: 1
24
+ )
25
+ Net::IMAP::SASL.authenticator(...)
57
26
  end
58
27
 
28
+ Net::IMAP.extend self
59
29
  end
60
30
 
61
- Net::IMAP.extend Net::IMAP::Authenticators
31
+ class Net::IMAP
32
+ PlainAuthenticator = SASL::PlainAuthenticator # :nodoc:
33
+ deprecate_constant :PlainAuthenticator
62
34
 
63
- require_relative "authenticators/plain"
64
-
65
- require_relative "authenticators/login"
66
- require_relative "authenticators/cram_md5"
67
- require_relative "authenticators/digest_md5"
68
- require_relative "authenticators/xoauth2"
35
+ XOauth2Authenticator = SASL::XOAuth2Authenticator # :nodoc:
36
+ deprecate_constant :XOauth2Authenticator
37
+ end
@@ -52,13 +52,20 @@ module Net
52
52
  end
53
53
 
54
54
  def send_string_data(str, tag = nil)
55
- case str
56
- when ""
55
+ if str.empty?
57
56
  put_string('""')
58
- when /[\x80-\xff\r\n]/n
59
- # literal
57
+ elsif str.match?(/[\r\n]/n)
58
+ # literal, because multiline
60
59
  send_literal(str, tag)
61
- when /[(){ \x00-\x1f\x7f%*"\\]/n
60
+ elsif !str.ascii_only?
61
+ if @utf8_strings
62
+ # quoted string
63
+ send_quoted_string(str)
64
+ else
65
+ # literal, because of non-ASCII bytes
66
+ send_literal(str, tag)
67
+ end
68
+ elsif str.match?(/[(){ \x00-\x1f\x7f%*"\\]/n)
62
69
  # quoted string
63
70
  send_quoted_string(str)
64
71
  else
@@ -67,7 +74,7 @@ module Net
67
74
  end
68
75
 
69
76
  def send_quoted_string(str)
70
- put_string('"' + str.gsub(/["\\]/n, "\\\\\\&") + '"')
77
+ put_string('"' + str.gsub(/["\\]/, "\\\\\\&") + '"')
71
78
  end
72
79
 
73
80
  def send_literal(str, tag = nil)
@@ -54,9 +54,9 @@ module Net
54
54
  # Net::IMAP does _not_ automatically encode and decode
55
55
  # mailbox names to and from UTF-7.
56
56
  def self.decode_utf7(s)
57
- return s.gsub(/&([^-]+)?-/n) {
58
- if $1
59
- ($1.tr(",", "/") + "===").unpack1("m").encode(Encoding::UTF_8, Encoding::UTF_16BE)
57
+ return s.gsub(/&([A-Za-z0-9+,]+)?-/n) {
58
+ if base64 = $1
59
+ (base64.tr(",", "/") + "===").unpack1("m").encode(Encoding::UTF_8, Encoding::UTF_16BE)
60
60
  else
61
61
  "&"
62
62
  end
@@ -0,0 +1,139 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Net
4
+ class IMAP < Protocol
5
+
6
+ # This module handles deprecated arguments to various Net::IMAP methods.
7
+ module DeprecatedClientOptions
8
+
9
+ # :call-seq:
10
+ # Net::IMAP.new(host, **options) # standard keyword options
11
+ # Net::IMAP.new(host, options) # obsolete hash options
12
+ # Net::IMAP.new(host, port) # obsolete port argument
13
+ # Net::IMAP.new(host, port, usessl, certs = nil, verify = true) # deprecated SSL arguments
14
+ #
15
+ # Translates Net::IMAP.new arguments for backward compatibility.
16
+ #
17
+ # ==== Obsolete arguments
18
+ #
19
+ # Using obsolete arguments does not a warning. Obsolete arguments will be
20
+ # deprecated by a future release.
21
+ #
22
+ # If a second positional argument is given and it is a hash (or is
23
+ # convertable via +#to_hash+), it is converted to keyword arguments.
24
+ #
25
+ # # Obsolete:
26
+ # Net::IMAP.new("imap.example.com", options_hash)
27
+ # # Use instead:
28
+ # Net::IMAP.new("imap.example.com", **options_hash)
29
+ #
30
+ # If a second positional argument is given and it is not a hash, it is
31
+ # converted to the +port+ keyword argument.
32
+ # # Obsolete:
33
+ # Net::IMAP.new("imap.example.com", 114433)
34
+ # # Use instead:
35
+ # Net::IMAP.new("imap.example.com", port: 114433)
36
+ #
37
+ # ==== Deprecated arguments
38
+ #
39
+ # Using deprecated arguments prints a warning. Convert to keyword
40
+ # arguments to avoid the warning. Deprecated arguments will be removed in
41
+ # a future release.
42
+ #
43
+ # If +usessl+ is false, +certs+, and +verify+ are ignored. When it true,
44
+ # all three arguments are converted to the +ssl+ keyword argument.
45
+ # Without +certs+ or +verify+, it is converted to <tt>ssl: true</tt>.
46
+ # # DEPRECATED:
47
+ # Net::IMAP.new("imap.example.com", nil, true) # => prints a warning
48
+ # # Use instead:
49
+ # Net::IMAP.new("imap.example.com", ssl: true)
50
+ #
51
+ # When +certs+ is a path to a directory, it is converted to <tt>ca_path:
52
+ # certs</tt>.
53
+ # # DEPRECATED:
54
+ # Net::IMAP.new("imap.example.com", nil, true, "/path/to/certs") # => prints a warning
55
+ # # Use instead:
56
+ # Net::IMAP.new("imap.example.com", ssl: {ca_path: "/path/to/certs"})
57
+ #
58
+ # When +certs+ is a path to a file, it is converted to <tt>ca_file:
59
+ # certs</tt>.
60
+ # # DEPRECATED:
61
+ # Net::IMAP.new("imap.example.com", nil, true, "/path/to/cert.pem") # => prints a warning
62
+ # # Use instead:
63
+ # Net::IMAP.new("imap.example.com", ssl: {ca_file: "/path/to/cert.pem"})
64
+ #
65
+ # When +verify+ is +false+, it is converted to <tt>verify_mode:
66
+ # OpenSSL::SSL::VERIFY_NONE</tt>.
67
+ # # DEPRECATED:
68
+ # Net::IMAP.new("imap.example.com", nil, true, nil, false) # => prints a warning
69
+ # # Use instead:
70
+ # Net::IMAP.new("imap.example.com", ssl: {verify_mode: OpenSSL::SSL::VERIFY_NONE})
71
+ #
72
+ def initialize(host, port_or_options = nil, *deprecated, **options)
73
+ if port_or_options.nil? && deprecated.empty?
74
+ super host, **options
75
+ elsif options.any?
76
+ # Net::IMAP.new(host, *__invalid__, **options)
77
+ raise ArgumentError, "Do not combine deprecated and keyword arguments"
78
+ elsif port_or_options.respond_to?(:to_hash) and deprecated.any?
79
+ # Net::IMAP.new(host, options, *__invalid__)
80
+ raise ArgumentError, "Do not use deprecated SSL params with options hash"
81
+ elsif port_or_options.respond_to?(:to_hash)
82
+ super host, **Hash.try_convert(port_or_options)
83
+ elsif deprecated.empty?
84
+ super host, port: port_or_options
85
+ elsif deprecated.shift
86
+ warn "DEPRECATED: Call Net::IMAP.new with keyword options", uplevel: 1
87
+ super host, port: port_or_options, ssl: create_ssl_params(*deprecated)
88
+ else
89
+ warn "DEPRECATED: Call Net::IMAP.new with keyword options", uplevel: 1
90
+ super host, port: port_or_options, ssl: false
91
+ end
92
+ end
93
+
94
+ # :call-seq:
95
+ # starttls(**options) # standard
96
+ # starttls(options = {}) # obsolete
97
+ # starttls(certs = nil, verify = true) # deprecated
98
+ #
99
+ # Translates Net::IMAP#starttls arguments for backward compatibility.
100
+ #
101
+ # Support for +certs+ and +verify+ will be dropped in a future release.
102
+ #
103
+ # See ::new for interpretation of +certs+ and +verify+.
104
+ def starttls(*deprecated, **options)
105
+ if deprecated.empty?
106
+ super(**options)
107
+ elsif options.any?
108
+ # starttls(*__invalid__, **options)
109
+ raise ArgumentError, "Do not combine deprecated and keyword options"
110
+ elsif deprecated.first.respond_to?(:to_hash) && deprecated.length > 1
111
+ # starttls(*__invalid__, **options)
112
+ raise ArgumentError, "Do not use deprecated verify param with options hash"
113
+ elsif deprecated.first.respond_to?(:to_hash)
114
+ super(**Hash.try_convert(deprecated.first))
115
+ else
116
+ warn "DEPRECATED: Call Net::IMAP#starttls with keyword options", uplevel: 1
117
+ super(**create_ssl_params(*deprecated))
118
+ end
119
+ end
120
+
121
+ private
122
+
123
+ def create_ssl_params(certs = nil, verify = true)
124
+ params = {}
125
+ if certs
126
+ if File.file?(certs)
127
+ params[:ca_file] = certs
128
+ elsif File.directory?(certs)
129
+ params[:ca_path] = certs
130
+ end
131
+ end
132
+ params[:verify_mode] =
133
+ verify ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
134
+ params
135
+ end
136
+
137
+ end
138
+ end
139
+ end
@@ -891,13 +891,6 @@ module Net
891
891
  # should use BodyTypeBasic.
892
892
  # BodyTypeMultipart:: for <tt>multipart/*</tt> parts
893
893
  #
894
- # ==== Deprecated BodyStructure classes
895
- # The following classes represent invalid server responses or parser bugs:
896
- # BodyTypeExtension:: parser bug: used for <tt>message/*</tt> where
897
- # BodyTypeBasic should have been used.
898
- # BodyTypeAttachment:: server bug: some servers sometimes return the
899
- # "Content-Disposition: attachment" data where the
900
- # entire body structure for a message part is expected.
901
894
  module BodyStructure
902
895
  end
903
896
 
@@ -914,6 +907,7 @@ module Net
914
907
  :param, :content_id,
915
908
  :description, :encoding, :size,
916
909
  :md5, :disposition, :language,
910
+ :location,
917
911
  :extension)
918
912
  include BodyStructure
919
913
 
@@ -1049,6 +1043,7 @@ module Net
1049
1043
  :description, :encoding, :size,
1050
1044
  :lines,
1051
1045
  :md5, :disposition, :language,
1046
+ :location,
1052
1047
  :extension)
1053
1048
  include BodyStructure
1054
1049
 
@@ -1094,6 +1089,7 @@ module Net
1094
1089
  :description, :encoding, :size,
1095
1090
  :envelope, :body, :lines,
1096
1091
  :md5, :disposition, :language,
1092
+ :location,
1097
1093
  :extension)
1098
1094
  include BodyStructure
1099
1095
 
@@ -1126,36 +1122,41 @@ module Net
1126
1122
  end
1127
1123
  end
1128
1124
 
1129
- # === WARNING
1130
- # BodyTypeAttachment represents a <tt>body-fld-dsp</tt> that is
1131
- # incorrectly in a position where the IMAP4rev1 grammar expects a nested
1132
- # +body+ structure.
1125
+ # BodyTypeAttachment is not used and will be removed in an upcoming release.
1133
1126
  #
1134
- # >>>
1135
- # \IMAP body structures are parenthesized lists and assign their fields
1136
- # positionally, so missing fields change the intepretation of all
1137
- # following fields. Buggy \IMAP servers sometimes leave fields missing
1138
- # rather than empty, which inevitably confuses parsers.
1139
- # BodyTypeAttachment was an attempt to parse a common type of buggy body
1140
- # structure without crashing.
1141
- #
1142
- # Currently, when Net::IMAP::ResponseParser sees "attachment" as the first
1143
- # entry in a <tt>body-type-1part</tt>, which is where the MIME type should
1144
- # be, it uses BodyTypeAttachment to capture the rest. "attachment" is not
1145
- # a valid MIME type, but _is_ a common <tt>Content-Disposition</tt>. What
1146
- # might have happened was that buggy server could not parse the message
1147
- # (which might have been incorrectly formatted) and output a
1148
- # <tt>body-type-dsp</tt> where a Net::IMAP::ResponseParser expected to see
1149
- # a +body+.
1150
- #
1151
- # A future release will replace this, probably with a ContentDisposition
1152
- # nested inside another body structure object, maybe BodyTypeBasic, or
1153
- # perhaps a new body structure class that represents any unparsable body
1154
- # structure.
1127
+ # === Bug Analysis
1128
+ #
1129
+ # \IMAP body structures are parenthesized lists and assign their fields
1130
+ # positionally, so missing fields change the intepretation of all
1131
+ # following fields. Additionally, different body types have a different
1132
+ # number of required fields, followed by optional "extension" fields.
1133
+ #
1134
+ # BodyTypeAttachment was previously returned when a "message/rfc822" part,
1135
+ # which should be sent as <tt>body-type-msg</tt> with ten required fields,
1136
+ # was actually sent as a <tt>body-type-basic</tt> with _seven_ required
1137
+ # fields.
1138
+ #
1139
+ # basic => type, subtype, param, id, desc, enc, octets, md5=nil, dsp=nil, lang=nil, loc=nil, *ext
1140
+ # msg => type, subtype, param, id, desc, enc, octets, envelope, body, lines, md5=nil, ...
1141
+ #
1142
+ # Normally, +envelope+ and +md5+ are incompatible, but Net::IMAP leniently
1143
+ # allowed buggy servers to send +NIL+ for +envelope+. As a result, when a
1144
+ # server sent a <tt>message/rfc822</tt> part with +NIL+ for +md5+ and a
1145
+ # non-<tt>NIL</tt> +dsp+, Net::IMAP mis-interpreted the
1146
+ # <tt>Content-Disposition</tt> as if it were a strange body type. In all
1147
+ # reported cases, the <tt>Content-Disposition</tt> was "attachment", so
1148
+ # BodyTypeAttachment was created as the workaround.
1149
+ #
1150
+ # === Current behavior
1151
+ #
1152
+ # When interpreted strictly, +envelope+ and +md5+ are incompatible. So the
1153
+ # current parsing algorithm peeks ahead after it has recieved the seventh
1154
+ # body field. If the next token is not the start of an +envelope+, we assume
1155
+ # the server has incorrectly sent us a <tt>body-type-basic</tt> and return
1156
+ # BodyTypeBasic. As a result, what was previously BodyTypeMessage#body =>
1157
+ # BodyTypeAttachment is now BodyTypeBasic#disposition => ContentDisposition.
1155
1158
  #
1156
1159
  class BodyTypeAttachment < Struct.new(:dsp_type, :_unused_, :param)
1157
- include BodyStructure
1158
-
1159
1160
  # *invalid for BodyTypeAttachment*
1160
1161
  def media_type
1161
1162
  warn(<<~WARN, uplevel: 1)
@@ -1190,11 +1191,14 @@ module Net
1190
1191
  end
1191
1192
  end
1192
1193
 
1194
+ deprecate_constant :BodyTypeAttachment
1195
+
1193
1196
  # Net::IMAP::BodyTypeMultipart represents body structures of messages and
1194
1197
  # message parts, when <tt>Content-Type</tt> is <tt>multipart/*</tt>.
1195
1198
  class BodyTypeMultipart < Struct.new(:media_type, :subtype,
1196
1199
  :parts,
1197
1200
  :param, :disposition, :language,
1201
+ :location,
1198
1202
  :extension)
1199
1203
  include BodyStructure
1200
1204
 
@@ -1265,23 +1269,24 @@ module Net
1265
1269
  end
1266
1270
  end
1267
1271
 
1268
- # === WARNING:
1272
+ # === Obsolete
1273
+ # BodyTypeExtension is not used and will be removed in an upcoming release.
1274
+ #
1269
1275
  # >>>
1270
- # BodyTypeExtension is (incorrectly) used for <tt>message/*</tt> parts
1276
+ # BodyTypeExtension was (incorrectly) used for <tt>message/*</tt> parts
1271
1277
  # (besides <tt>message/rfc822</tt>, which correctly uses BodyTypeMessage).
1272
1278
  #
1273
- # A future release will replace this class with:
1274
- # * BodyTypeMessage for <tt>message/rfc822</tt> and <tt>message/global</tt>
1275
- # * BodyTypeBasic for any other <tt>message/*</tt>
1279
+ # Net::IMAP now (correctly) parses all message types (other than
1280
+ # <tt>message/rfc822</tt> or <tt>message/global</tt>) as BodyTypeBasic.
1276
1281
  class BodyTypeExtension < Struct.new(:media_type, :subtype,
1277
1282
  :params, :content_id,
1278
1283
  :description, :encoding, :size)
1279
- include BodyStructure
1280
-
1281
1284
  def multipart?
1282
1285
  return false
1283
1286
  end
1284
1287
  end
1285
1288
 
1289
+ deprecate_constant :BodyTypeExtension
1290
+
1286
1291
  end
1287
1292
  end
@@ -0,0 +1,230 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Net
4
+ class IMAP < Protocol
5
+ class ResponseParser
6
+ # basic utility methods for parsing.
7
+ #
8
+ # (internal API, subject to change)
9
+ module ParserUtils # :nodoc:
10
+
11
+ module Generator
12
+
13
+ LOOKAHEAD = "(@token ||= next_token)"
14
+ SHIFT_TOKEN = "(@token = nil)"
15
+
16
+ # we can skip lexer for single character matches, as a shortcut
17
+ def def_char_matchers(name, char, token)
18
+ match_name = name.match(/\A[A-Z]/) ? "#{name}!" : name
19
+ char = char.dump
20
+ class_eval <<~RUBY, __FILE__, __LINE__ + 1
21
+ # frozen_string_literal: true
22
+
23
+ # force use of #next_token; no string peeking
24
+ def lookahead_#{name}?
25
+ #{LOOKAHEAD}&.symbol == #{token}
26
+ end
27
+
28
+ # use token or string peek
29
+ def peek_#{name}?
30
+ @token ? @token.symbol == #{token} : @str[@pos] == #{char}
31
+ end
32
+
33
+ # like accept(token_symbols); returns token or nil
34
+ def #{name}?
35
+ if @token&.symbol == #{token}
36
+ #{SHIFT_TOKEN}
37
+ #{char}
38
+ elsif !@token && @str[@pos] == #{char}
39
+ @pos += 1
40
+ #{char}
41
+ end
42
+ end
43
+
44
+ # like match(token_symbols); returns token or raises parse_error
45
+ def #{match_name}
46
+ if @token&.symbol == #{token}
47
+ #{SHIFT_TOKEN}
48
+ #{char}
49
+ elsif !@token && @str[@pos] == #{char}
50
+ @pos += 1
51
+ #{char}
52
+ else
53
+ parse_error("unexpected %s (expected %p)",
54
+ @token&.symbol || @str[@pos].inspect, #{char})
55
+ end
56
+ end
57
+ RUBY
58
+ end
59
+
60
+ # TODO: move coersion to the token.value method?
61
+ def def_token_matchers(name, *token_symbols, coerce: nil, send: nil)
62
+ match_name = name.match(/\A[A-Z]/) ? "#{name}!" : name
63
+
64
+ if token_symbols.size == 1
65
+ token = token_symbols.first
66
+ matcher = "token&.symbol == %p" % [token]
67
+ desc = token
68
+ else
69
+ matcher = "%p.include? token&.symbol" % [token_symbols]
70
+ desc = token_symbols.join(" or ")
71
+ end
72
+
73
+ value = "(token.value)"
74
+ value = coerce.to_s + value if coerce
75
+ value = [value, send].join(".") if send
76
+
77
+ raise_parse_error = <<~RUBY
78
+ parse_error("unexpected %s (expected #{desc})", token&.symbol)
79
+ RUBY
80
+
81
+ class_eval <<~RUBY, __FILE__, __LINE__ + 1
82
+ # frozen_string_literal: true
83
+
84
+ # lookahead version of match, returning the value
85
+ def lookahead_#{name}!
86
+ token = #{LOOKAHEAD}
87
+ if #{matcher}
88
+ #{value}
89
+ else
90
+ #{raise_parse_error}
91
+ end
92
+ end
93
+
94
+ def #{name}?
95
+ token = #{LOOKAHEAD}
96
+ if #{matcher}
97
+ #{SHIFT_TOKEN}
98
+ #{value}
99
+ end
100
+ end
101
+
102
+ def #{match_name}
103
+ token = #{LOOKAHEAD}
104
+ if #{matcher}
105
+ #{SHIFT_TOKEN}
106
+ #{value}
107
+ else
108
+ #{raise_parse_error}
109
+ end
110
+ end
111
+ RUBY
112
+
113
+ end
114
+
115
+ end
116
+
117
+ private
118
+
119
+ # TODO: after checking the lookahead, use a regexp for remaining chars.
120
+ # That way a loop isn't needed.
121
+ def combine_adjacent(*tokens)
122
+ result = "".b
123
+ while token = accept(*tokens)
124
+ result << token.value
125
+ end
126
+ if result.empty?
127
+ parse_error('unexpected token %s (expected %s)',
128
+ lookahead.symbol, tokens.join(" or "))
129
+ end
130
+ result
131
+ end
132
+
133
+ def match(*args)
134
+ token = lookahead
135
+ unless args.include?(token.symbol)
136
+ parse_error('unexpected token %s (expected %s)',
137
+ token.symbol.id2name,
138
+ args.collect {|i| i.id2name}.join(" or "))
139
+ end
140
+ shift_token
141
+ token
142
+ end
143
+
144
+ # like match, but does not raise error on failure.
145
+ #
146
+ # returns and shifts token on successful match
147
+ # returns nil and leaves @token unshifted on no match
148
+ def accept(*args)
149
+ token = lookahead
150
+ if args.include?(token.symbol)
151
+ shift_token
152
+ token
153
+ end
154
+ end
155
+
156
+ # To be used conditionally:
157
+ # assert_no_lookahead if Net::IMAP.debug
158
+ def assert_no_lookahead
159
+ @token.nil? or
160
+ parse_error("assertion failed: expected @token.nil?, actual %s: %p",
161
+ @token.symbol, @token.value)
162
+ end
163
+
164
+ # like accept, without consuming the token
165
+ def lookahead?(*symbols)
166
+ @token if symbols.include?((@token ||= next_token)&.symbol)
167
+ end
168
+
169
+ def lookahead
170
+ @token ||= next_token
171
+ end
172
+
173
+ def peek_str?(str)
174
+ assert_no_lookahead if Net::IMAP.debug
175
+ @str[@pos, str.length] == str
176
+ end
177
+
178
+ def peek_re(re)
179
+ assert_no_lookahead if Net::IMAP.debug
180
+ re.match(@str, @pos)
181
+ end
182
+
183
+ def accept_re(re)
184
+ assert_no_lookahead if Net::IMAP.debug
185
+ re.match(@str, @pos) and @pos = $~.end(0)
186
+ $~
187
+ end
188
+
189
+ def match_re(re, name)
190
+ assert_no_lookahead if Net::IMAP.debug
191
+ if re.match(@str, @pos)
192
+ @pos = $~.end(0)
193
+ $~
194
+ else
195
+ parse_error("invalid #{name}")
196
+ end
197
+ end
198
+
199
+ def shift_token
200
+ @token = nil
201
+ end
202
+
203
+ def parse_error(fmt, *args)
204
+ msg = format(fmt, *args)
205
+ if IMAP.debug
206
+ local_path = File.dirname(__dir__)
207
+ tok = @token ? "%s: %p" % [@token.symbol, @token.value] : "nil"
208
+ warn "%s %s: %s" % [self.class, __method__, msg]
209
+ warn " tokenized : %s" % [@str[...@pos].dump]
210
+ warn " remaining : %s" % [@str[@pos..].dump]
211
+ warn " @lex_state: %s" % [@lex_state]
212
+ warn " @pos : %d" % [@pos]
213
+ warn " @token : %s" % [tok]
214
+ caller_locations(1..20).each_with_index do |cloc, idx|
215
+ next unless cloc.path&.start_with?(local_path)
216
+ warn " caller[%2d]: %-30s (%s:%d)" % [
217
+ idx,
218
+ cloc.base_label,
219
+ File.basename(cloc.path, ".rb"),
220
+ cloc.lineno
221
+ ]
222
+ end
223
+ end
224
+ raise ResponseParseError, msg
225
+ end
226
+
227
+ end
228
+ end
229
+ end
230
+ end