mongo 2.14.0 → 2.14.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/lib/mongo/background_thread.rb +11 -0
  4. data/lib/mongo/cluster/sdam_flow.rb +14 -0
  5. data/lib/mongo/cluster.rb +0 -26
  6. data/lib/mongo/collection/view/iterable.rb +16 -6
  7. data/lib/mongo/collection.rb +2 -0
  8. data/lib/mongo/cursor.rb +10 -0
  9. data/lib/mongo/database.rb +14 -2
  10. data/lib/mongo/grid/fs_bucket.rb +37 -37
  11. data/lib/mongo/operation/parallel_scan/command.rb +1 -2
  12. data/lib/mongo/operation/shared/read_preference_supported.rb +38 -36
  13. data/lib/mongo/operation/shared/sessions_supported.rb +3 -2
  14. data/lib/mongo/protocol/msg.rb +2 -2
  15. data/lib/mongo/protocol/query.rb +11 -11
  16. data/lib/mongo/server/connection_pool.rb +0 -2
  17. data/lib/mongo/server.rb +0 -14
  18. data/lib/mongo/server_selector/secondary_preferred.rb +2 -7
  19. data/lib/mongo/srv/monitor.rb +0 -11
  20. data/lib/mongo/version.rb +1 -1
  21. data/spec/integration/query_cache_spec.rb +45 -0
  22. data/spec/integration/sdam_error_handling_spec.rb +1 -1
  23. data/spec/integration/sdam_events_spec.rb +3 -5
  24. data/spec/integration/secondary_reads_spec.rb +102 -0
  25. data/spec/mongo/cluster_spec.rb +2 -18
  26. data/spec/mongo/index/view_spec.rb +4 -2
  27. data/spec/mongo/operation/read_preference_legacy_spec.rb +9 -19
  28. data/spec/mongo/operation/read_preference_op_msg_spec.rb +3 -3
  29. data/spec/mongo/server/app_metadata_shared.rb +33 -7
  30. data/spec/mongo/server_selector/secondary_preferred_spec.rb +6 -6
  31. data/spec/runners/transactions/operation.rb +13 -2
  32. data/spec/shared/bin/get-mongodb-download-url +17 -0
  33. data/spec/shared/lib/mrss/cluster_config.rb +221 -0
  34. data/spec/shared/lib/mrss/constraints.rb +43 -0
  35. data/spec/shared/lib/mrss/docker_runner.rb +265 -0
  36. data/spec/shared/lib/mrss/lite_constraints.rb +16 -0
  37. data/spec/shared/lib/mrss/server_version_registry.rb +115 -0
  38. data/spec/shared/lib/mrss/spec_organizer.rb +3 -0
  39. data/spec/shared/lib/mrss/utils.rb +15 -0
  40. data/spec/shared/share/Dockerfile.erb +231 -0
  41. data/spec/shared/shlib/distro.sh +73 -0
  42. data/spec/shared/shlib/server.sh +290 -0
  43. data/spec/shared/shlib/set_env.sh +128 -0
  44. data/spec/solo/clean_exit_spec.rb +21 -0
  45. data/spec/support/client_registry.rb +8 -4
  46. data/spec/support/client_registry_macros.rb +4 -4
  47. data/spec/support/spec_config.rb +12 -0
  48. data/spec/support/spec_setup.rb +48 -38
  49. data.tar.gz.sig +0 -0
  50. metadata +1005 -983
  51. metadata.gz.sig +0 -0
@@ -78,8 +78,8 @@ describe Mongo::ServerSelector::SecondaryPreferred do
78
78
 
79
79
  context 'tag sets not provided' do
80
80
 
81
- it 'returns nil' do
82
- expect(selector.to_mongos).to be_nil
81
+ it 'returns secondaryPreferred' do
82
+ selector.to_mongos.should == {mode: 'secondaryPreferred'}
83
83
  end
84
84
  end
85
85
 
@@ -89,8 +89,8 @@ describe Mongo::ServerSelector::SecondaryPreferred do
89
89
  { :mode => 'secondaryPreferred' }
90
90
  end
91
91
 
92
- it 'returns nil' do
93
- expect(selector.to_mongos).to be_nil
92
+ it 'returns secondaryPreferred' do
93
+ selector.to_mongos.should == {mode: 'secondaryPreferred'}
94
94
  end
95
95
  end
96
96
 
@@ -120,8 +120,8 @@ describe Mongo::ServerSelector::SecondaryPreferred do
120
120
  context 'hedge not provided' do
121
121
  let(:hedge) { nil }
122
122
 
123
- it 'returns nil' do
124
- expect(selector.to_mongos).to be_nil
123
+ it 'returns secondaryPreferred' do
124
+ selector.to_mongos.should == {mode: 'secondaryPreferred'}
125
125
  end
126
126
  end
127
127
 
@@ -191,8 +191,19 @@ module Mongo
191
191
 
192
192
  def assert_event_count(client, context)
193
193
  events = _select_events(context)
194
- unless events.length == arguments['count']
195
- raise "Exppected #{arguments['count']} #{arguments['event']} events, but have #{events.length}"
194
+ if arguments['event'] == 'ServerMarkedUnknownEvent'
195
+ # We publish SDAM events from both regular and push monitors.
196
+ # This means sometimes there are two ServerMarkedUnknownEvent
197
+ # events published for the same server transition.
198
+ # Allow actual event count to be at least the expected event count
199
+ # in case there are multiple transitions in a single test.
200
+ unless events.length >= arguments['count']
201
+ raise "Expected #{arguments['count']} #{arguments['event']} events, but have #{events.length}"
202
+ end
203
+ else
204
+ unless events.length == arguments['count']
205
+ raise "Expected #{arguments['count']} #{arguments['event']} events, but have #{events.length}"
206
+ end
196
207
  end
197
208
  end
198
209
 
@@ -0,0 +1,17 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ desired_version, arch = ARGV
4
+ if arch.nil?
5
+ STDERR.puts "Usage: get-mongodb-download-url desired-version arch"
6
+ exit 1
7
+ end
8
+
9
+ $: << File.join(File.dirname(__FILE__), '../lib')
10
+ require 'mrss/server_version_registry'
11
+
12
+ begin
13
+ puts Mrss::ServerVersionRegistry.new(desired_version, arch).download_url
14
+ rescue Mrss::ServerVersionRegistry::Error => exc
15
+ STDERR.puts "Error: #{exc}"
16
+ exit 2
17
+ end
@@ -0,0 +1,221 @@
1
+ # frozen_string_literal: true
2
+ # encoding: utf-8
3
+
4
+ # ClusterConfig requires ClientRegistry class provided by the host project.
5
+
6
+ require 'singleton'
7
+
8
+ module Mrss
9
+ class ClusterConfig
10
+ include Singleton
11
+ include RSpec::Core::Pending
12
+
13
+ def single_server?
14
+ determine_cluster_config
15
+ @single_server
16
+ end
17
+
18
+ def replica_set_name
19
+ determine_cluster_config
20
+ @replica_set_name
21
+ end
22
+
23
+ def server_version
24
+ determine_cluster_config
25
+ @server_version
26
+ end
27
+
28
+ def enterprise?
29
+ determine_cluster_config
30
+ @enterprise
31
+ end
32
+
33
+ def short_server_version
34
+ server_version.split('.')[0..1].join('.')
35
+ end
36
+
37
+ def fcv
38
+ determine_cluster_config
39
+ @fcv
40
+ end
41
+
42
+ # Per https://jira.mongodb.org/browse/SERVER-39052, working with FCV
43
+ # in sharded topologies is annoying. Also, FCV doesn't exist in servers
44
+ # less than 3.4. This method returns FCV on 3.4+ servers when in single
45
+ # or RS topologies, and otherwise returns the major.minor server version.
46
+ def fcv_ish
47
+ if server_version.nil?
48
+ raise "Deployment server version not known - check that connection to deployment succeeded"
49
+ end
50
+
51
+ if server_version >= '3.4' && topology != :sharded
52
+ fcv
53
+ else
54
+ if short_server_version == '4.1'
55
+ '4.2'
56
+ else
57
+ short_server_version
58
+ end
59
+ end
60
+ end
61
+
62
+ # @return [ Mongo::Address ] The address of the primary in the deployment.
63
+ def primary_address
64
+ determine_cluster_config
65
+ @primary_address
66
+ end
67
+
68
+ def primary_address_str
69
+ determine_cluster_config
70
+ @primary_address.seed
71
+ end
72
+
73
+ def primary_address_host
74
+ both = primary_address_str
75
+ both.split(':').first
76
+ end
77
+
78
+ def primary_address_port
79
+ both = primary_address_str
80
+ both.split(':')[1] || 27017
81
+ end
82
+
83
+ def primary_description
84
+ determine_cluster_config
85
+ @primary_description
86
+ end
87
+
88
+ def server_parameters
89
+ determine_cluster_config
90
+ @server_parameters
91
+ end
92
+
93
+ # Try running a command on the admin database to see if the mongod was
94
+ # started with auth.
95
+ def auth_enabled?
96
+ if @auth_enabled.nil?
97
+ @auth_enabled = begin
98
+ basic_client.use(:admin).command(getCmdLineOpts: 1).first["argv"].include?("--auth")
99
+ rescue => e
100
+ e.message =~ /(not authorized)|(unauthorized)|(no users authenticated)|(requires authentication)/
101
+ end
102
+ end
103
+ @auth_enabled
104
+ end
105
+
106
+ def topology
107
+ determine_cluster_config
108
+ @topology
109
+ end
110
+
111
+ def storage_engine
112
+ @storage_engine ||= begin
113
+ # 2.6 does not have wired tiger
114
+ if short_server_version == '2.6'
115
+ :mmapv1
116
+ else
117
+ client = ClientRegistry.instance.global_client('root_authorized')
118
+ if topology == :sharded
119
+ shards = client.use(:admin).command(listShards: 1).first
120
+ if shards['shards'].empty?
121
+ raise 'Shards are empty'
122
+ end
123
+ shard = shards['shards'].first
124
+ address_str = shard['host'].sub(/^.*\//, '').sub(/,.*/, '')
125
+ client = ClusterTools.instance.direct_client(address_str,
126
+ SpecConfig.instance.test_options.merge(SpecConfig.instance.auth_options).merge(connect: :direct))
127
+ end
128
+ rv = client.use(:admin).command(serverStatus: 1).first
129
+ rv = rv['storageEngine']['name']
130
+ rv_map = {
131
+ 'wiredTiger' => :wired_tiger,
132
+ 'mmapv1' => :mmapv1,
133
+ }
134
+ rv_map[rv] || rv
135
+ end
136
+ end
137
+ end
138
+
139
+ # This method returns an alternate address for connecting to the configured
140
+ # deployment. For example, if the replica set is configured with nodes at
141
+ # of localhost:27017 and so on, this method will return 127.0.0.:27017.
142
+ #
143
+ # Note that the "alternate" refers to replica set configuration, not the
144
+ # addresses specified in test suite configuration. If the deployment topology
145
+ # is not a replica set, "alternate" refers to test suite configuration as
146
+ # this is the only configuration available.
147
+ def alternate_address
148
+ @alternate_address ||= begin
149
+ address = primary_address_host
150
+ str = case address
151
+ when '127.0.0.1'
152
+ 'localhost'
153
+ when /^(\d+\.){3}\d+$/
154
+ skip 'This test requires a hostname or 127.0.0.1 as address'
155
+ else
156
+ # We don't know if mongod is listening on ipv4 or ipv6, in principle.
157
+ # Our tests use ipv4, so hardcode that for now.
158
+ # To support both we need to try both addresses which will make this
159
+ # test more complicated.
160
+ #
161
+ # JRuby chokes on primary_address_port as the port (e.g. 27017).
162
+ # Since the port does not actually matter, use a common port like 80.
163
+ resolved_address = Addrinfo.getaddrinfo(address, 80, Socket::PF_INET).first.ip_address
164
+ if resolved_address.include?(':')
165
+ "[#{resolved_address}]"
166
+ else
167
+ resolved_address
168
+ end
169
+ end + ":#{primary_address_port}"
170
+ Mongo::Address.new(str)
171
+ end
172
+ end
173
+
174
+ private
175
+
176
+ def determine_cluster_config
177
+ return if @primary_address
178
+
179
+ # Run all commands to figure out the cluster configuration from the same
180
+ # client. This is somewhat wasteful when running a single test, but reduces
181
+ # test runtime for the suite overall because all commands are sent on the
182
+ # same connection rather than each command connecting to the cluster by
183
+ # itself.
184
+ client = ClientRegistry.instance.global_client('root_authorized')
185
+
186
+ primary = client.cluster.next_primary
187
+ @primary_address = primary.address
188
+ @primary_description = primary.description
189
+ @replica_set_name = client.cluster.topology.replica_set_name
190
+
191
+ @topology ||= begin
192
+ topology = client.cluster.topology.class.name.sub(/.*::/, '')
193
+ topology = topology.gsub(/([A-Z])/) { |match| '_' + match.downcase }.sub(/^_/, '')
194
+ if topology =~ /^replica_set/
195
+ topology = 'replica_set'
196
+ end
197
+ topology.to_sym
198
+ end
199
+
200
+ @single_server = client.cluster.servers_list.length == 1
201
+
202
+ build_info = client.database.command(buildInfo: 1).first
203
+
204
+ @server_version = build_info['version']
205
+ @enterprise = build_info['modules'] && build_info['modules'].include?('enterprise')
206
+
207
+ @server_parameters = client.use(:admin).command(getParameter: '*').first
208
+
209
+ if @topology != :sharded && short_server_version >= '3.4'
210
+ rv = @server_parameters['featureCompatibilityVersion']
211
+ @fcv = rv['version'] || rv
212
+ end
213
+ end
214
+
215
+ def basic_client
216
+ # Do not cache the result here so that if the client gets closed,
217
+ # client registry reconnects it in subsequent tests
218
+ ClientRegistry.instance.global_client('basic')
219
+ end
220
+ end
221
+ end
@@ -147,6 +147,33 @@ module Mrss
147
147
  end
148
148
  end
149
149
 
150
+ def require_no_snappy_compression
151
+ before(:all) do
152
+ compressors = SpecConfig.instance.compressors
153
+ if compressors && compressors.include?('snappy')
154
+ skip "Snappy compression is enabled"
155
+ end
156
+ end
157
+ end
158
+
159
+ def require_zstd_compression
160
+ before(:all) do
161
+ compressors = SpecConfig.instance.compressors
162
+ unless compressors && compressors.include?('zstd')
163
+ skip "Zstd compression is not enabled"
164
+ end
165
+ end
166
+ end
167
+
168
+ def require_no_zstd_compression
169
+ before(:all) do
170
+ compressors = SpecConfig.instance.compressors
171
+ if compressors && compressors.include?('zstd')
172
+ skip "Zstd compression is enabled"
173
+ end
174
+ end
175
+ end
176
+
150
177
  def require_no_compression
151
178
  before(:all) do
152
179
  if SpecConfig.instance.compressors
@@ -299,5 +326,21 @@ module Mrss
299
326
  end
300
327
  end
301
328
  end
329
+
330
+ def require_required_api_version
331
+ before(:all) do
332
+ unless ENV['API_VERSION_REQUIRED'] == '1'
333
+ skip 'Set API_VERSION_REQUIRED=1 to run this test'
334
+ end
335
+ end
336
+ end
337
+
338
+ def require_no_required_api_version
339
+ before(:all) do
340
+ if ENV['API_VERSION_REQUIRED'] == '1'
341
+ skip 'Cannot have API_VERSION_REQUIRED=1 to run this test'
342
+ end
343
+ end
344
+ end
302
345
  end
303
346
  end
@@ -0,0 +1,265 @@
1
+ # frozen_string_literal: true
2
+ # encoding: utf-8
3
+
4
+ require 'optparse'
5
+ require 'erb'
6
+ autoload :Dotenv, 'dotenv'
7
+
8
+ module Mrss
9
+ autoload :ServerVersionRegistry, 'mrss/server_version_registry'
10
+
11
+ class DockerRunner
12
+ def initialize(**opts)
13
+ # These options are required:
14
+ opts.fetch(:image_tag)
15
+ opts.fetch(:dockerfile_path)
16
+ opts.fetch(:default_script)
17
+ opts.fetch(:project_lib_subdir)
18
+
19
+ @options = opts
20
+ end
21
+
22
+ attr_reader :options
23
+
24
+ def run
25
+ process_arguments
26
+ unless @options[:exec_only]
27
+ create_dockerfile
28
+ create_image
29
+ end
30
+ if @options[:mongo_only]
31
+ run_deployment
32
+ else
33
+ run_tests
34
+ end
35
+ end
36
+
37
+ private
38
+
39
+ def process_arguments
40
+ #@options = {}
41
+ OptionParser.new do |opts|
42
+ opts.banner = "Usage: test-on-docker [-d distro] [evergreen_key=value ...]"
43
+
44
+ opts.on("-a", "--add-env=PATH", "Load environment variables from PATH in .env format") do |path|
45
+ @options[:extra_env] ||= {}
46
+ unless File.exist?(path)
47
+ raise "-a option references nonexistent file #{path}"
48
+ end
49
+ Dotenv.parse(path).each do |k, v|
50
+ @options[:extra_env][k] = v
51
+ end
52
+ end
53
+
54
+ opts.on("-d", "--distro=DISTRO", "Distro to use") do |v|
55
+ @options[:distro] = v
56
+ end
57
+
58
+ opts.on('-e', '--exec-only', 'Execute tests using existing Dockerfile (for offline user)') do |v|
59
+ @options[:exec_only] = v
60
+ end
61
+
62
+ opts.on('-m', '--mongo-only=PORT', 'Start the MongoDB deployment and expose it to host on ports starting with PORT') do |v|
63
+ @options[:mongo_only] = v.to_i
64
+ end
65
+
66
+ opts.on('-p', '--preload', 'Preload Ruby toolchain and server binaries in docker') do |v|
67
+ @options[:preload] = v
68
+ end
69
+
70
+ opts.on('-s', '--script=SCRIPT', 'Test script to invoke') do |v|
71
+ @options[:script] = v
72
+ end
73
+
74
+ opts.on('-i', '--interactive', 'Interactive mode - disable per-test timeouts') do |v|
75
+ @options[:interactive] = v
76
+ end
77
+ end.parse!
78
+
79
+ @env = Hash[ARGV.map do |arg|
80
+ arg.split('=', 2)
81
+ end]
82
+
83
+ @env['RVM_RUBY'] ||= 'ruby-2.7'
84
+ unless ruby =~ /^j?ruby-/
85
+ raise "RVM_RUBY option is not in expected format: #{ruby}"
86
+ end
87
+
88
+ @env['MONGODB_VERSION'] ||= '4.4'
89
+ end
90
+
91
+ def create_dockerfile
92
+ template_path = File.join(File.dirname(__FILE__), '../../share/Dockerfile.erb')
93
+ result = ERB.new(File.read(template_path)).result(binding)
94
+ File.open(dockerfile_path, 'w') do |f|
95
+ f << result
96
+ end
97
+ end
98
+
99
+ def image_tag
100
+ options.fetch(:image_tag)
101
+ end
102
+
103
+ def dockerfile_path
104
+ options.fetch(:dockerfile_path)
105
+ end
106
+
107
+ def create_image
108
+ run_command(['docker', 'build',
109
+ '-t', image_tag,
110
+ '-f', dockerfile_path,
111
+ '.'])
112
+ end
113
+
114
+ BASE_TEST_COMMAND = %w(docker run -i --tmpfs /tmpfs:exec).freeze
115
+
116
+ def run_tests
117
+ run_command(BASE_TEST_COMMAND + tty_arg + extra_env + [image_tag] +
118
+ script.split(/\s+/))
119
+ end
120
+
121
+ def run_deployment
122
+ run_command(BASE_TEST_COMMAND + tty_arg + extra_env + [
123
+ '-e', %q`TEST_CMD=watch -x bash -c "ps awwxu |egrep 'mongo|ocsp'"`,
124
+ '-e', 'BIND_ALL=true',
125
+ ] + port_forwards + [image_tag] + script.split(/\s+/))
126
+ end
127
+
128
+ def tty_arg
129
+ tty = File.open('/dev/stdin') do |f|
130
+ f.isatty
131
+ end
132
+ if tty
133
+ %w(-t --init)
134
+ else
135
+ []
136
+ end
137
+ end
138
+
139
+ def extra_env
140
+ if @options[:extra_env]
141
+ @options[:extra_env].map do |k, v|
142
+ # Here the value must not be escaped
143
+ ['-e', "#{k}=#{v}"]
144
+ end.flatten
145
+ else
146
+ []
147
+ end
148
+ end
149
+
150
+ def port_forwards
151
+ args = (0...num_exposed_ports).map do |i|
152
+ host_port = @options[:mongo_only] + i
153
+ container_port = 27017 + i
154
+ ['-p', "#{host_port}:#{container_port}"]
155
+ end.flatten
156
+
157
+ if @env['OCSP_ALGORITHM'] && !@env['OCSP_VERIFIER']
158
+ args += %w(-p 8100:8100)
159
+ end
160
+
161
+ args
162
+ end
163
+
164
+ def run_command(cmd)
165
+ if pid = fork
166
+ Process.wait(pid)
167
+ unless $?.exitstatus == 0
168
+ raise "Process exited with code #{$?.exitstatus}"
169
+ end
170
+ else
171
+ exec(*cmd)
172
+ end
173
+ end
174
+
175
+ def distro
176
+ @options[:distro] || 'ubuntu1604'
177
+ end
178
+
179
+ BASE_IMAGES = {
180
+ 'debian81' => 'debian:jessie',
181
+ 'debian92' => 'debian:stretch',
182
+ 'ubuntu1404' => 'ubuntu:trusty',
183
+ 'ubuntu1604' => 'ubuntu:xenial',
184
+ 'ubuntu1804' => 'ubuntu:bionic',
185
+ 'rhel62' => 'centos:6',
186
+ 'rhel70' => 'centos:7',
187
+ }.freeze
188
+
189
+ def base_image
190
+ BASE_IMAGES[distro] or raise "Unknown distro: #{distro}"
191
+ end
192
+
193
+ def ruby
194
+ @env['RVM_RUBY']
195
+ end
196
+
197
+ def ruby_head?
198
+ ruby == 'ruby-head'
199
+ end
200
+
201
+ def server_version
202
+ @env['MONGODB_VERSION']
203
+ end
204
+
205
+ def script
206
+ @options[:script] || options.fetch(:default_script)
207
+ end
208
+
209
+ def debian?
210
+ distro =~ /debian|ubuntu/
211
+ end
212
+
213
+ def preload?
214
+ !!@options[:preload]
215
+ end
216
+
217
+ def interactive?
218
+ !!@options[:interactive]
219
+ end
220
+
221
+ def project_lib_subdir
222
+ options.fetch(:project_lib_subdir)
223
+ end
224
+
225
+ def server_download_url
226
+ @server_download_url ||= ServerVersionRegistry.new(server_version, distro).download_url
227
+ end
228
+
229
+ def libmongocrypt_path
230
+ case distro
231
+ when /ubuntu1604/
232
+ "./ubuntu1604/nocrypto/lib64/libmongocrypt.so"
233
+ when /ubuntu1804/
234
+ "./ubuntu1804-64/nocrypto/lib64/libmongocrypt.so"
235
+ when /debian92/
236
+ "./debian92/nocrypto/lib64/libmongocrypt.so"
237
+ else
238
+ raise "This script does not support running FLE tests on #{distro}. Use ubuntu1604, ubuntu1804 or debian92 instead"
239
+ end
240
+ end
241
+
242
+ def expose?
243
+ !!@options[:mongo_only]
244
+ end
245
+
246
+ def fle?
247
+ %w(1 true yes).include?(@env['FLE']&.downcase)
248
+ end
249
+
250
+ def num_exposed_ports
251
+ case @env['TOPOLOGY'] || 'standalone'
252
+ when 'standalone'
253
+ 1
254
+ when 'replica-set'
255
+ 3
256
+ when 'sharded-cluster'
257
+ if @env['SINGLE_MONGOS']
258
+ 1
259
+ else
260
+ 2
261
+ end
262
+ end
263
+ end
264
+ end
265
+ end