mongo 2.13.0.rc1 → 2.13.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/lib/mongo/address.rb +1 -1
  4. data/lib/mongo/auth/aws/request.rb +27 -3
  5. data/lib/mongo/client.rb +48 -2
  6. data/lib/mongo/collection.rb +21 -12
  7. data/lib/mongo/database/view.rb +1 -1
  8. data/lib/mongo/database.rb +14 -2
  9. data/lib/mongo/error/invalid_server_auth_host.rb +22 -0
  10. data/lib/mongo/error/operation_failure.rb +5 -5
  11. data/lib/mongo/error.rb +2 -0
  12. data/lib/mongo/grid/fs_bucket.rb +37 -37
  13. data/lib/mongo/index/view.rb +3 -0
  14. data/lib/mongo/operation/collections_info/command.rb +5 -0
  15. data/lib/mongo/operation/collections_info/result.rb +16 -1
  16. data/lib/mongo/operation/parallel_scan/command.rb +1 -2
  17. data/lib/mongo/operation/shared/read_preference_supported.rb +38 -36
  18. data/lib/mongo/operation/shared/sessions_supported.rb +3 -2
  19. data/lib/mongo/protocol/message.rb +11 -2
  20. data/lib/mongo/protocol/msg.rb +22 -3
  21. data/lib/mongo/protocol/query.rb +47 -11
  22. data/lib/mongo/server/app_metadata.rb +27 -3
  23. data/lib/mongo/server/connection_base.rb +35 -11
  24. data/lib/mongo/server_selector/secondary_preferred.rb +2 -7
  25. data/lib/mongo/version.rb +1 -1
  26. data/spec/integration/bson_symbol_spec.rb +4 -2
  27. data/spec/integration/bulk_write_spec.rb +19 -0
  28. data/spec/integration/client_authentication_options_spec.rb +37 -0
  29. data/spec/integration/client_side_encryption/auto_encryption_bulk_writes_spec.rb +9 -5
  30. data/spec/integration/sdam_error_handling_spec.rb +18 -1
  31. data/spec/integration/sdam_events_spec.rb +8 -7
  32. data/spec/integration/secondary_reads_spec.rb +102 -0
  33. data/spec/integration/size_limit_spec.rb +20 -6
  34. data/spec/lite_spec_helper.rb +1 -1
  35. data/spec/mongo/auth/aws/request_region_spec.rb +42 -0
  36. data/spec/mongo/auth/aws/request_spec.rb +32 -32
  37. data/spec/mongo/client_construction_spec.rb +123 -0
  38. data/spec/mongo/client_encryption_spec.rb +16 -10
  39. data/spec/mongo/crypt/data_key_context_spec.rb +1 -1
  40. data/spec/mongo/database_spec.rb +64 -0
  41. data/spec/mongo/index/view_spec.rb +150 -2
  42. data/spec/mongo/operation/read_preference_legacy_spec.rb +9 -19
  43. data/spec/mongo/operation/read_preference_op_msg_spec.rb +3 -3
  44. data/spec/mongo/server/app_metadata_shared.rb +114 -8
  45. data/spec/mongo/server_selector/secondary_preferred_spec.rb +6 -6
  46. data/spec/runners/transactions/operation.rb +13 -2
  47. data/spec/shared/LICENSE +20 -0
  48. data/spec/shared/bin/get-mongodb-download-url +17 -0
  49. data/spec/shared/lib/mrss/child_process_helper.rb +80 -0
  50. data/spec/shared/lib/mrss/cluster_config.rb +221 -0
  51. data/spec/shared/lib/mrss/constraints.rb +346 -0
  52. data/spec/shared/lib/mrss/docker_runner.rb +265 -0
  53. data/spec/shared/lib/mrss/lite_constraints.rb +191 -0
  54. data/spec/shared/lib/mrss/server_version_registry.rb +115 -0
  55. data/spec/shared/lib/mrss/spec_organizer.rb +152 -0
  56. data/spec/shared/lib/mrss/utils.rb +15 -0
  57. data/spec/shared/share/Dockerfile.erb +231 -0
  58. data/spec/shared/shlib/distro.sh +73 -0
  59. data/spec/shared/shlib/server.sh +290 -0
  60. data/spec/shared/shlib/set_env.sh +128 -0
  61. data/spec/support/client_registry.rb +8 -4
  62. data/spec/support/client_registry_macros.rb +14 -5
  63. data/spec/support/spec_config.rb +12 -0
  64. data/spec/support/spec_setup.rb +48 -38
  65. data.tar.gz.sig +3 -1
  66. metadata +1005 -974
  67. metadata.gz.sig +0 -0
  68. data/spec/integration/size_limit_spec.rb~12e1e9c4f... RUBY-2242 Fix zlib compression (#2021) +0 -98
@@ -37,8 +37,7 @@ module Mongo
37
37
  read_concern)
38
38
  end
39
39
  sel[:maxTimeMS] = max_time_ms if max_time_ms
40
- update_selector_for_read_pref(sel, connection)
41
- sel
40
+ add_read_preference_legacy(sel, connection)
42
41
  end
43
42
 
44
43
  def message(connection)
@@ -36,47 +36,44 @@ module Mongo
36
36
  #
37
37
  # @since 2.0.0
38
38
  def options(connection)
39
- add_slave_ok_flag_maybe(super, connection)
39
+ options = super
40
+ if add_slave_ok_flag?(connection)
41
+ flags = options[:flags]&.dup || []
42
+ flags << :slave_ok
43
+ options = options.merge(flags: flags)
44
+ end
45
+ options
40
46
  end
41
47
 
42
- # Adds :slave_ok flag to options based on the read preference specified
43
- # in the operation or implied by the topology that the connection's
44
- # server is a part of.
48
+ # Whether to add the :slave_ok flag to the request based on the
49
+ # read preference specified in the operation or implied by the topology
50
+ # that the connection's server is a part of.
45
51
  #
46
- # @param [ Hash ] options The options calculated so far.
47
52
  # @param [ Server::Connection ] connection The connection that the
48
53
  # operation will be executed on.
49
54
  #
50
- # @return [ Hash ] The new options.
51
- def add_slave_ok_flag_maybe(options, connection)
52
- add_flag =
53
- # https://github.com/mongodb/specifications/blob/master/source/server-selection/server-selection.rst#topology-type-single
54
- if connection.description.standalone?
55
- # Read preference is never sent to standalones.
56
- false
57
- elsif connection.server.cluster.single?
58
- # In Single topology the driver forces primaryPreferred read
59
- # preference mode (via the slave_ok flag, in case of old servers)
60
- # so that the query is satisfied.
61
- true
62
- else
63
- # In replica sets and sharded clusters, read preference is passed
64
- # to the server if one is specified by the application, and there
65
- # is no default.
66
- read && read.slave_ok?
67
- end
68
-
69
- if add_flag
70
- options= options.dup
71
- (options[:flags] ||= []) << :slave_ok
55
+ # @return [ true | false ] Whether the :slave_ok flag should be added.
56
+ def add_slave_ok_flag?(connection)
57
+ # https://github.com/mongodb/specifications/blob/master/source/server-selection/server-selection.rst#topology-type-single
58
+ if connection.description.standalone?
59
+ # Read preference is never sent to standalones.
60
+ false
61
+ elsif connection.server.cluster.single?
62
+ # In Single topology the driver forces primaryPreferred read
63
+ # preference mode (via the slave_ok flag, in case of old servers)
64
+ # so that the query is satisfied.
65
+ true
66
+ else
67
+ # In replica sets and sharded clusters, read preference is passed
68
+ # to the server if one is specified by the application, and there
69
+ # is no default.
70
+ read && read.slave_ok? || false
72
71
  end
73
-
74
- options
75
72
  end
76
73
 
77
74
  def command(connection)
78
75
  sel = super
79
- update_selector_for_read_pref(sel, connection)
76
+ add_read_preference_legacy(sel, connection)
80
77
  end
81
78
 
82
79
  # Adds $readPreference field to the command document.
@@ -95,14 +92,19 @@ module Mongo
95
92
  # operation will be executed on.
96
93
  #
97
94
  # @return [ Hash ] New command document to send to the server.
98
- def update_selector_for_read_pref(sel, connection)
95
+ def add_read_preference_legacy(sel, connection)
99
96
  if read && connection.description.mongos? && read_pref = read.to_mongos
100
- Mongo::Lint.validate_camel_case_read_preference(read_pref)
101
- sel = sel[:$query] ? sel : {:$query => sel}
102
- sel = sel.merge(:$readPreference => read_pref)
103
- else
104
- sel
97
+ # If the read preference contains only mode and mode is secondary
98
+ # preferred and we are sending to a pre-OP_MSG server, this read
99
+ # preference is indicated by the :slave_ok wire protocol flag
100
+ # and $readPreference command parameter isn't sent.
101
+ if read_pref != {mode: 'secondaryPreferred'}
102
+ Mongo::Lint.validate_camel_case_read_preference(read_pref)
103
+ sel = sel[:$query] ? sel : {:$query => sel}
104
+ sel = sel.merge(:$readPreference => read_pref)
105
+ end
105
106
  end
107
+ sel
106
108
  end
107
109
  end
108
110
  end
@@ -170,9 +170,10 @@ module Mongo
170
170
  elsif connection.description.mongos?
171
171
  # When server is a mongos:
172
172
  # - $readPreference is never sent when mode is 'primary'
173
- # - When mode is 'secondaryPreferred' $readPreference is only sent
174
- # when a non-mode field (i.e. tag_sets) is present
175
173
  # - Otherwise $readPreference is sent
174
+ # When mode is 'secondaryPreferred' $readPreference is currently
175
+ # required to only be sent when a non-mode field (i.e. tag_sets)
176
+ # is present, but this causes wrong behavior (DRIVERS-1642).
176
177
  if read
177
178
  doc = read.to_mongos
178
179
  if doc
@@ -177,10 +177,19 @@ module Mongo
177
177
  #
178
178
  # @param buffer [String] buffer where the message should be inserted
179
179
  # @return [String] buffer containing the serialized message
180
- def serialize(buffer = BSON::ByteBuffer.new, max_bson_size = nil)
180
+ def serialize(buffer = BSON::ByteBuffer.new, max_bson_size = nil, bson_overhead = nil)
181
+ max_size =
182
+ if max_bson_size && bson_overhead
183
+ max_bson_size + bson_overhead
184
+ elsif max_bson_size
185
+ max_bson_size
186
+ else
187
+ nil
188
+ end
189
+
181
190
  start = buffer.length
182
191
  serialize_header(buffer)
183
- serialize_fields(buffer, max_bson_size)
192
+ serialize_fields(buffer, max_size)
184
193
  buffer.replace_int32(start, buffer.length - start)
185
194
  end
186
195
 
@@ -43,8 +43,8 @@ module Mongo
43
43
  # Msg.new([:more_to_come], {}, { ismaster: 1 },
44
44
  # { type: 1, payload: { identifier: 'documents', sequence: [..] } })
45
45
  #
46
- # @param [ Array<Symbol> ] flags The flag bits. Current supported values
47
- # are :more_to_come and :checksum_present.
46
+ # @param [ Array<Symbol> ] flags The flag bits. Currently supported
47
+ # values are :more_to_come and :checksum_present.
48
48
  # @param [ Hash ] options The options.
49
49
  # @param [ BSON::Document, Hash ] main_document The document that will
50
50
  # become the payload type 0 section. Can contain global args as they
@@ -146,7 +146,9 @@ module Mongo
146
146
  # @return [ BSON::ByteBuffer ] buffer containing the serialized message.
147
147
  #
148
148
  # @since 2.5.0
149
- def serialize(buffer = BSON::ByteBuffer.new, max_bson_size = nil)
149
+ def serialize(buffer = BSON::ByteBuffer.new, max_bson_size = nil, bson_overhead = nil)
150
+ validate_document_size!(max_bson_size)
151
+
150
152
  super
151
153
  add_check_sum(buffer)
152
154
  buffer
@@ -275,6 +277,23 @@ module Mongo
275
277
 
276
278
  private
277
279
 
280
+ # Validate that the documents in this message are all smaller than the
281
+ # maxBsonObjectSize. If not, raise an exception.
282
+ def validate_document_size!(max_bson_size)
283
+ max_bson_size ||= Mongo::Server::ConnectionBase::DEFAULT_MAX_BSON_OBJECT_SIZE
284
+
285
+ contains_too_large_document = @sections.any? do |section|
286
+ section[:type] == 1 &&
287
+ section[:payload][:sequence].any? do |document|
288
+ document.to_bson.length > max_bson_size
289
+ end
290
+ end
291
+
292
+ if contains_too_large_document
293
+ raise Error::MaxBSONSize.new('The document exceeds maximum allowed BSON object size after serialization')
294
+ end
295
+ end
296
+
278
297
  def command
279
298
  @command ||= if @main_document
280
299
  @main_document.dup.tap do |cmd|
@@ -46,18 +46,18 @@ module Mongo
46
46
  # @example Find all user ids.
47
47
  # Query.new('xgen', 'users', {}, :fields => {:id => 1})
48
48
  #
49
- # @param database [String, Symbol] The database to query.
50
- # @param collection [String, Symbol] The collection to query.
51
- # @param selector [Hash] The query selector.
52
- # @param options [Hash] The additional query options.
49
+ # @param [ String, Symbol ] database The database to query.
50
+ # @param [ String, Symbol ] collection The collection to query.
51
+ # @param [ Hash ] selector The query selector.
52
+ # @param [ Hash ] options The additional query options.
53
53
  #
54
- # @option options :project [Hash] The projection.
55
- # @option options :skip [Integer] The number of documents to skip.
56
- # @option options :limit [Integer] The number of documents to return.
57
- # @option options :flags [Array] The flags for the query message.
58
- #
59
- # Supported flags: +:tailable_cursor+, +:slave_ok+, +:oplog_replay+,
60
- # +:no_cursor_timeout+, +:await_data+, +:exhaust+, +:partial+
54
+ # @option options [ Array<Symbol> ] :flags The flag bits.
55
+ # Currently supported values are :await_data, :exhaust,
56
+ # :no_cursor_timeout, :oplog_replay, :partial, :slave_ok,
57
+ # :tailable_cursor.
58
+ # @option options [ Integer ] :limit The number of documents to return.
59
+ # @option options [ Hash ] :project The projection.
60
+ # @option options [ Integer ] :skip The number of documents to skip.
61
61
  def initialize(database, collection, selector, options = {})
62
62
  @database = database
63
63
  @namespace = "#{database}.#{collection}"
@@ -115,12 +115,48 @@ module Mongo
115
115
  compress_if_possible(selector.keys.first, compressor, zlib_compression_level)
116
116
  end
117
117
 
118
+ # Serializes message into bytes that can be sent on the wire.
119
+ #
120
+ # @param [ BSON::ByteBuffer ] buffer where the message should be inserted.
121
+ # @param [ Integer ] max_bson_size The maximum bson object size.
122
+ #
123
+ # @return [ BSON::ByteBuffer ] buffer containing the serialized message.
124
+ def serialize(buffer = BSON::ByteBuffer.new, max_bson_size = nil, bson_overhead = nil)
125
+ validate_document_size!(max_bson_size)
126
+
127
+ super
128
+ end
129
+
118
130
  protected
119
131
 
120
132
  attr_reader :upconverter
121
133
 
122
134
  private
123
135
 
136
+ # Validate that the documents in this message are all smaller than the
137
+ # maxBsonObjectSize. If not, raise an exception.
138
+ def validate_document_size!(max_bson_size)
139
+ max_bson_size ||= Mongo::Server::ConnectionBase::DEFAULT_MAX_BSON_OBJECT_SIZE
140
+
141
+ documents = if @selector.key?(:documents)
142
+ @selector[:documents]
143
+ elsif @selector.key?(:deletes)
144
+ @selector[:deletes]
145
+ elsif @selector.key?(:updates)
146
+ @selector[:updates]
147
+ else
148
+ []
149
+ end
150
+
151
+ contains_too_large_document = documents.any? do |doc|
152
+ doc.to_bson.length > max_bson_size
153
+ end
154
+
155
+ if contains_too_large_document
156
+ raise Error::MaxBSONSize.new('The document exceeds maximum allowed BSON object size after serialization')
157
+ end
158
+ end
159
+
124
160
  # The operation code required to specify a Query message.
125
161
  # @return [Fixnum] the operation code.
126
162
  #
@@ -65,12 +65,17 @@ module Mongo
65
65
  # the metadata printed to the mongod logs upon establishing a connection
66
66
  # in server versions >= 3.4.
67
67
  # @option options [ String ] :user The user name.
68
+ # @option options [ Array<Hash> ] :wrapping_libraries Information about
69
+ # libraries such as ODMs that are wrapping the driver. Specify the
70
+ # lower level libraries first. Allowed hash keys: :name, :version,
71
+ # :platform.
68
72
  #
69
73
  # @since 2.4.0
70
74
  def initialize(options)
71
75
  @app_name = options[:app_name].to_s if options[:app_name]
72
76
  @platform = options[:platform]
73
77
  @compressors = options[:compressors] || []
78
+ @wrapping_libraries = options[:wrapping_libraries]
74
79
 
75
80
  if options[:user] && !options[:auth_mech]
76
81
  auth_db = options[:auth_source] || 'admin'
@@ -78,6 +83,10 @@ module Mongo
78
83
  end
79
84
  end
80
85
 
86
+ # @return [ Array<Hash> | nil ] Information about libraries wrapping
87
+ # the driver.
88
+ attr_reader :wrapping_libraries
89
+
81
90
  # Get the bytes of the ismaster message including this metadata.
82
91
  #
83
92
  # @api private
@@ -140,9 +149,17 @@ module Mongo
140
149
  end
141
150
 
142
151
  def driver_doc
152
+ names = [DRIVER_NAME]
153
+ versions = [Mongo::VERSION]
154
+ if wrapping_libraries
155
+ wrapping_libraries.each do |library|
156
+ names << library[:name] || ''
157
+ versions << library[:version] || ''
158
+ end
159
+ end
143
160
  {
144
- name: DRIVER_NAME,
145
- version: Mongo::VERSION
161
+ name: names.join('|'),
162
+ version: versions.join('|'),
146
163
  }
147
164
  end
148
165
 
@@ -175,12 +192,19 @@ module Mongo
175
192
  ruby_versions = ["Ruby #{RUBY_VERSION}"]
176
193
  platforms = [RUBY_PLATFORM]
177
194
  end
178
- [
195
+ platform = [
179
196
  @platform,
180
197
  *ruby_versions,
181
198
  *platforms,
182
199
  RbConfig::CONFIG['build'],
183
200
  ].compact.join(', ')
201
+ platforms = [platform]
202
+ if wrapping_libraries
203
+ wrapping_libraries.each do |library|
204
+ platforms << library[:platform] || ''
205
+ end
206
+ end
207
+ platforms.join('|')
184
208
  end
185
209
  end
186
210
  end
@@ -186,9 +186,6 @@ module Mongo
186
186
  end
187
187
 
188
188
  def serialize(message, client, buffer = BSON::ByteBuffer.new)
189
- start_size = 0
190
- final_message = message.maybe_compress(compressor, options[:zlib_compression_level])
191
-
192
189
  # Driver specifications only mandate the fixed 16MiB limit for
193
190
  # serialized BSON documents. However, the server returns its
194
191
  # active serialized BSON document size limit in the ismaster response,
@@ -207,18 +204,45 @@ module Mongo
207
204
  if message.bulk_write?
208
205
  # Make the new maximum size equal to the specified reduced size
209
206
  # limit plus the 16KiB overhead allowance.
210
- max_bson_size = REDUCED_MAX_BSON_SIZE + MAX_BSON_COMMAND_OVERHEAD
207
+ max_bson_size = REDUCED_MAX_BSON_SIZE
211
208
  end
212
- else
213
- max_bson_size += MAX_BSON_COMMAND_OVERHEAD
214
209
  end
215
210
 
216
- final_message.serialize(buffer, max_bson_size)
217
- if max_message_size &&
218
- (buffer.length - start_size) > max_message_size
219
- then
220
- raise Error::MaxMessageSize.new(max_message_size)
211
+ # RUBY-2234: It is necessary to check that the message size does not
212
+ # exceed the maximum bson object size before compressing and serializing
213
+ # the final message.
214
+ #
215
+ # This is to avoid the case where the user performs a bulk write
216
+ # larger than 16MiB which, when compressed, becomes smaller than 16MiB.
217
+ # If the driver does not split the bulk writes prior to compression,
218
+ # the entire operation will be sent to the server, which will raise an
219
+ # error because the uncompressed operation exceeds the maximum bson size.
220
+ #
221
+ # To address this problem, we serialize the message prior to compression
222
+ # and raise an exception if the serialized message exceeds the maximum
223
+ # bson size.
224
+ if max_message_size
225
+ # Create a separate buffer that contains the un-compressed message
226
+ # for the purpose of checking its size. Write any pre-existing contents
227
+ # from the original buffer into the temporary one.
228
+ temp_buffer = BSON::ByteBuffer.new
229
+
230
+ # TODO: address the fact that this line mutates the buffer.
231
+ temp_buffer.put_bytes(buffer.get_bytes(buffer.length))
232
+
233
+ message.serialize(temp_buffer, max_bson_size, MAX_BSON_COMMAND_OVERHEAD)
234
+ if temp_buffer.length > max_message_size
235
+ raise Error::MaxMessageSize.new(max_message_size)
236
+ end
221
237
  end
238
+
239
+ # RUBY-2335: When the un-compressed message is smaller than the maximum
240
+ # bson size limit, the message will be serialized twice. The operations
241
+ # layer should be refactored to allow compression on an already-
242
+ # serialized message.
243
+ final_message = message.maybe_compress(compressor, options[:zlib_compression_level])
244
+ final_message.serialize(buffer, max_bson_size, MAX_BSON_COMMAND_OVERHEAD)
245
+
222
246
  buffer
223
247
  end
224
248
  end
@@ -86,13 +86,8 @@ module Mongo
86
86
  #
87
87
  # @since 2.0.0
88
88
  def to_mongos
89
- if tag_sets.empty? && max_staleness.nil? && hedge.nil?
90
- # The server preference is not sent to mongos as part of the query
91
- # selector if there are no tag sets, for maximum backwards compatibility.
92
- nil
93
- else
94
- to_doc
95
- end
89
+ # Always send the read preference to mongos: DRIVERS-1642.
90
+ to_doc
96
91
  end
97
92
 
98
93
  private
data/lib/mongo/version.rb CHANGED
@@ -17,5 +17,5 @@ module Mongo
17
17
  # The current version of the driver.
18
18
  #
19
19
  # @since 2.0.0
20
- VERSION = '2.13.0.rc1'.freeze
20
+ VERSION = '2.13.3'.freeze
21
21
  end
@@ -25,8 +25,10 @@ describe 'Symbol encoding to BSON' do
25
25
  end
26
26
 
27
27
  it 'round-trips symbol values using the same byte buffer' do
28
- if BSON::Environment.jruby?
29
- pending 'https://jira.mongodb.org/browse/RUBY-2128'
28
+ if BSON::Environment.jruby? && (BSON::VERSION.split('.').map(&:to_i) <=> [4, 11, 0]) < 0
29
+ skip 'This test is only relevant to bson versions that increment ByteBuffer '\
30
+ 'read and write positions separately in JRuby, as implemented in ' \
31
+ 'bson version 4.11.0. For more information, see https://jira.mongodb.org/browse/RUBY-2128'
30
32
  end
31
33
 
32
34
  Hash.from_bson(hash.to_bson).should == hash
@@ -0,0 +1,19 @@
1
+ require 'spec_helper'
2
+
3
+ describe 'Bulk writes' do
4
+ before do
5
+ authorized_collection.drop
6
+ end
7
+
8
+ context 'when bulk write is larger than 48MB' do
9
+ let(:operations) do
10
+ [ { insert_one: { text: 'a' * 1000 * 1000 } } ] * 48
11
+ end
12
+
13
+ it 'succeeds' do
14
+ expect do
15
+ authorized_collection.bulk_write(operations)
16
+ end.not_to raise_error
17
+ end
18
+ end
19
+ end