bson 4.12.1 → 4.13.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (79) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/README.md +4 -7
  4. data/ext/bson/bson-native.h +2 -0
  5. data/ext/bson/init.c +9 -0
  6. data/ext/bson/read.c +29 -0
  7. data/lib/bson/active_support.rb +1 -0
  8. data/lib/bson/array.rb +2 -1
  9. data/lib/bson/binary.rb +5 -3
  10. data/lib/bson/boolean.rb +2 -1
  11. data/lib/bson/code.rb +2 -1
  12. data/lib/bson/code_with_scope.rb +2 -1
  13. data/lib/bson/config.rb +1 -0
  14. data/lib/bson/date.rb +1 -0
  15. data/lib/bson/date_time.rb +1 -0
  16. data/lib/bson/db_pointer.rb +2 -1
  17. data/lib/bson/dbref.rb +125 -0
  18. data/lib/bson/decimal128/builder.rb +18 -15
  19. data/lib/bson/decimal128.rb +10 -9
  20. data/lib/bson/document.rb +14 -0
  21. data/lib/bson/environment.rb +1 -0
  22. data/lib/bson/error.rb +7 -0
  23. data/lib/bson/ext_json.rb +16 -11
  24. data/lib/bson/false_class.rb +2 -1
  25. data/lib/bson/float.rb +20 -31
  26. data/lib/bson/hash.rb +15 -6
  27. data/lib/bson/int32.rb +3 -2
  28. data/lib/bson/int64.rb +3 -2
  29. data/lib/bson/integer.rb +3 -2
  30. data/lib/bson/json.rb +1 -0
  31. data/lib/bson/max_key.rb +3 -2
  32. data/lib/bson/min_key.rb +3 -2
  33. data/lib/bson/nil_class.rb +2 -1
  34. data/lib/bson/object.rb +1 -0
  35. data/lib/bson/object_id.rb +4 -3
  36. data/lib/bson/open_struct.rb +1 -0
  37. data/lib/bson/regexp.rb +17 -6
  38. data/lib/bson/registry.rb +1 -0
  39. data/lib/bson/specialized.rb +1 -0
  40. data/lib/bson/string.rb +3 -2
  41. data/lib/bson/symbol.rb +2 -1
  42. data/lib/bson/time.rb +4 -3
  43. data/lib/bson/time_with_zone.rb +1 -0
  44. data/lib/bson/timestamp.rb +3 -2
  45. data/lib/bson/true_class.rb +2 -1
  46. data/lib/bson/undefined.rb +2 -1
  47. data/lib/bson/version.rb +2 -1
  48. data/lib/bson.rb +6 -4
  49. data/spec/README.md +14 -0
  50. data/spec/bson/dbref_spec.rb +461 -0
  51. data/spec/bson/document_spec.rb +7 -1
  52. data/spec/bson/ext_json_parse_spec.rb +37 -0
  53. data/spec/bson/int64_spec.rb +4 -24
  54. data/spec/bson/raw_spec.rb +7 -1
  55. data/spec/bson/regexp_spec.rb +52 -0
  56. data/spec/runners/common_driver.rb +1 -1
  57. data/spec/shared/LICENSE +20 -0
  58. data/spec/shared/bin/get-mongodb-download-url +17 -0
  59. data/spec/shared/lib/mrss/child_process_helper.rb +80 -0
  60. data/spec/shared/lib/mrss/cluster_config.rb +221 -0
  61. data/spec/shared/lib/mrss/constraints.rb +346 -0
  62. data/spec/shared/lib/mrss/docker_runner.rb +265 -0
  63. data/spec/shared/lib/mrss/lite_constraints.rb +191 -0
  64. data/spec/shared/lib/mrss/server_version_registry.rb +115 -0
  65. data/spec/shared/lib/mrss/spec_organizer.rb +152 -0
  66. data/spec/shared/lib/mrss/utils.rb +15 -0
  67. data/spec/shared/share/Dockerfile.erb +231 -0
  68. data/spec/shared/shlib/distro.sh +73 -0
  69. data/spec/shared/shlib/server.sh +290 -0
  70. data/spec/shared/shlib/set_env.sh +128 -0
  71. data/spec/spec_helper.rb +9 -1
  72. data/spec/spec_tests/data/corpus/binary.json +5 -0
  73. data/spec/spec_tests/data/corpus/dbref.json +21 -1
  74. data/spec/spec_tests/data/corpus/document.json +4 -0
  75. data/spec/spec_tests/data/corpus/regex.json +2 -2
  76. data/spec/spec_tests/data/corpus/top.json +20 -9
  77. data.tar.gz.sig +0 -0
  78. metadata +123 -90
  79. metadata.gz.sig +0 -0
@@ -0,0 +1,265 @@
1
+ # frozen_string_literal: true
2
+ # encoding: utf-8
3
+
4
+ require 'optparse'
5
+ require 'erb'
6
+ autoload :Dotenv, 'dotenv'
7
+
8
+ module Mrss
9
+ autoload :ServerVersionRegistry, 'mrss/server_version_registry'
10
+
11
+ class DockerRunner
12
+ def initialize(**opts)
13
+ # These options are required:
14
+ opts.fetch(:image_tag)
15
+ opts.fetch(:dockerfile_path)
16
+ opts.fetch(:default_script)
17
+ opts.fetch(:project_lib_subdir)
18
+
19
+ @options = opts
20
+ end
21
+
22
+ attr_reader :options
23
+
24
+ def run
25
+ process_arguments
26
+ unless @options[:exec_only]
27
+ create_dockerfile
28
+ create_image
29
+ end
30
+ if @options[:mongo_only]
31
+ run_deployment
32
+ else
33
+ run_tests
34
+ end
35
+ end
36
+
37
+ private
38
+
39
+ def process_arguments
40
+ #@options = {}
41
+ OptionParser.new do |opts|
42
+ opts.banner = "Usage: test-on-docker [-d distro] [evergreen_key=value ...]"
43
+
44
+ opts.on("-a", "--add-env=PATH", "Load environment variables from PATH in .env format") do |path|
45
+ @options[:extra_env] ||= {}
46
+ unless File.exist?(path)
47
+ raise "-a option references nonexistent file #{path}"
48
+ end
49
+ Dotenv.parse(path).each do |k, v|
50
+ @options[:extra_env][k] = v
51
+ end
52
+ end
53
+
54
+ opts.on("-d", "--distro=DISTRO", "Distro to use") do |v|
55
+ @options[:distro] = v
56
+ end
57
+
58
+ opts.on('-e', '--exec-only', 'Execute tests using existing Dockerfile (for offline user)') do |v|
59
+ @options[:exec_only] = v
60
+ end
61
+
62
+ opts.on('-m', '--mongo-only=PORT', 'Start the MongoDB deployment and expose it to host on ports starting with PORT') do |v|
63
+ @options[:mongo_only] = v.to_i
64
+ end
65
+
66
+ opts.on('-p', '--preload', 'Preload Ruby toolchain and server binaries in docker') do |v|
67
+ @options[:preload] = v
68
+ end
69
+
70
+ opts.on('-s', '--script=SCRIPT', 'Test script to invoke') do |v|
71
+ @options[:script] = v
72
+ end
73
+
74
+ opts.on('-i', '--interactive', 'Interactive mode - disable per-test timeouts') do |v|
75
+ @options[:interactive] = v
76
+ end
77
+ end.parse!
78
+
79
+ @env = Hash[ARGV.map do |arg|
80
+ arg.split('=', 2)
81
+ end]
82
+
83
+ @env['RVM_RUBY'] ||= 'ruby-2.7'
84
+ unless ruby =~ /^j?ruby-/
85
+ raise "RVM_RUBY option is not in expected format: #{ruby}"
86
+ end
87
+
88
+ @env['MONGODB_VERSION'] ||= '4.4'
89
+ end
90
+
91
+ def create_dockerfile
92
+ template_path = File.join(File.dirname(__FILE__), '../../share/Dockerfile.erb')
93
+ result = ERB.new(File.read(template_path)).result(binding)
94
+ File.open(dockerfile_path, 'w') do |f|
95
+ f << result
96
+ end
97
+ end
98
+
99
+ def image_tag
100
+ options.fetch(:image_tag)
101
+ end
102
+
103
+ def dockerfile_path
104
+ options.fetch(:dockerfile_path)
105
+ end
106
+
107
+ def create_image
108
+ run_command(['docker', 'build',
109
+ '-t', image_tag,
110
+ '-f', dockerfile_path,
111
+ '.'])
112
+ end
113
+
114
+ BASE_TEST_COMMAND = %w(docker run -i --tmpfs /tmpfs:exec).freeze
115
+
116
+ def run_tests
117
+ run_command(BASE_TEST_COMMAND + tty_arg + extra_env + [image_tag] +
118
+ script.split(/\s+/))
119
+ end
120
+
121
+ def run_deployment
122
+ run_command(BASE_TEST_COMMAND + tty_arg + extra_env + [
123
+ '-e', %q`TEST_CMD=watch -x bash -c "ps awwxu |egrep 'mongo|ocsp'"`,
124
+ '-e', 'BIND_ALL=true',
125
+ ] + port_forwards + [image_tag] + script.split(/\s+/))
126
+ end
127
+
128
+ def tty_arg
129
+ tty = File.open('/dev/stdin') do |f|
130
+ f.isatty
131
+ end
132
+ if tty
133
+ %w(-t --init)
134
+ else
135
+ []
136
+ end
137
+ end
138
+
139
+ def extra_env
140
+ if @options[:extra_env]
141
+ @options[:extra_env].map do |k, v|
142
+ # Here the value must not be escaped
143
+ ['-e', "#{k}=#{v}"]
144
+ end.flatten
145
+ else
146
+ []
147
+ end
148
+ end
149
+
150
+ def port_forwards
151
+ args = (0...num_exposed_ports).map do |i|
152
+ host_port = @options[:mongo_only] + i
153
+ container_port = 27017 + i
154
+ ['-p', "#{host_port}:#{container_port}"]
155
+ end.flatten
156
+
157
+ if @env['OCSP_ALGORITHM'] && !@env['OCSP_VERIFIER']
158
+ args += %w(-p 8100:8100)
159
+ end
160
+
161
+ args
162
+ end
163
+
164
+ def run_command(cmd)
165
+ if pid = fork
166
+ Process.wait(pid)
167
+ unless $?.exitstatus == 0
168
+ raise "Process exited with code #{$?.exitstatus}"
169
+ end
170
+ else
171
+ exec(*cmd)
172
+ end
173
+ end
174
+
175
+ def distro
176
+ @options[:distro] || 'ubuntu1604'
177
+ end
178
+
179
+ BASE_IMAGES = {
180
+ 'debian81' => 'debian:jessie',
181
+ 'debian92' => 'debian:stretch',
182
+ 'ubuntu1404' => 'ubuntu:trusty',
183
+ 'ubuntu1604' => 'ubuntu:xenial',
184
+ 'ubuntu1804' => 'ubuntu:bionic',
185
+ 'rhel62' => 'centos:6',
186
+ 'rhel70' => 'centos:7',
187
+ }.freeze
188
+
189
+ def base_image
190
+ BASE_IMAGES[distro] or raise "Unknown distro: #{distro}"
191
+ end
192
+
193
+ def ruby
194
+ @env['RVM_RUBY']
195
+ end
196
+
197
+ def ruby_head?
198
+ ruby == 'ruby-head'
199
+ end
200
+
201
+ def server_version
202
+ @env['MONGODB_VERSION']
203
+ end
204
+
205
+ def script
206
+ @options[:script] || options.fetch(:default_script)
207
+ end
208
+
209
+ def debian?
210
+ distro =~ /debian|ubuntu/
211
+ end
212
+
213
+ def preload?
214
+ !!@options[:preload]
215
+ end
216
+
217
+ def interactive?
218
+ !!@options[:interactive]
219
+ end
220
+
221
+ def project_lib_subdir
222
+ options.fetch(:project_lib_subdir)
223
+ end
224
+
225
+ def server_download_url
226
+ @server_download_url ||= ServerVersionRegistry.new(server_version, distro).download_url
227
+ end
228
+
229
+ def libmongocrypt_path
230
+ case distro
231
+ when /ubuntu1604/
232
+ "./ubuntu1604/nocrypto/lib64/libmongocrypt.so"
233
+ when /ubuntu1804/
234
+ "./ubuntu1804-64/nocrypto/lib64/libmongocrypt.so"
235
+ when /debian92/
236
+ "./debian92/nocrypto/lib64/libmongocrypt.so"
237
+ else
238
+ raise "This script does not support running FLE tests on #{distro}. Use ubuntu1604, ubuntu1804 or debian92 instead"
239
+ end
240
+ end
241
+
242
+ def expose?
243
+ !!@options[:mongo_only]
244
+ end
245
+
246
+ def fle?
247
+ %w(1 true yes).include?(@env['FLE']&.downcase)
248
+ end
249
+
250
+ def num_exposed_ports
251
+ case @env['TOPOLOGY'] || 'standalone'
252
+ when 'standalone'
253
+ 1
254
+ when 'replica-set'
255
+ 3
256
+ when 'sharded-cluster'
257
+ if @env['SINGLE_MONGOS']
258
+ 1
259
+ else
260
+ 2
261
+ end
262
+ end
263
+ end
264
+ end
265
+ end
@@ -0,0 +1,191 @@
1
+ # frozen_string_literal: true
2
+ # encoding: utf-8
3
+
4
+ module Mrss
5
+ module LiteConstraints
6
+
7
+ # Constrain tests that use TimeoutInterrupt to MRI (and Unix).
8
+ def require_mri
9
+ before(:all) do
10
+ unless SpecConfig.instance.mri?
11
+ skip "MRI required, we have #{SpecConfig.instance.platform}"
12
+ end
13
+ end
14
+ end
15
+
16
+ def require_jruby
17
+ before(:all) do
18
+ unless BSON::Environment.jruby?
19
+ skip "JRuby required, we have #{SpecConfig.instance.platform}"
20
+ end
21
+ end
22
+ end
23
+
24
+ # This is for marking tests that fail on JRuby that should
25
+ # in principle work (as opposed to being fundamentally incompatible
26
+ # with JRuby).
27
+ # Often times these failures happen only in Evergreen.
28
+ def fails_on_jruby(version=nil)
29
+ before(:all) do
30
+ if BSON::Environment.jruby?
31
+ if version
32
+ min_parts = version.split('.').map(&:to_i)
33
+ actual_parts = JRUBY_VERSION.split('.').map(&:to_i)[0...min_parts.length]
34
+ actual = actual_parts.join('.')
35
+ if actual <= version
36
+ skip "Fails on jruby through #{version}"
37
+ end
38
+ else
39
+ skip "Fails on jruby"
40
+ end
41
+ end
42
+ end
43
+ end
44
+
45
+ # Indicates that the respective test uses the internet in some capacity,
46
+ # for example the test resolves SRV DNS records.
47
+ def require_external_connectivity
48
+ before(:all) do
49
+ if ENV['EXTERNAL_DISABLED']
50
+ skip "Test requires external connectivity"
51
+ end
52
+ end
53
+ end
54
+
55
+ def require_mongo_kerberos
56
+ before(:all) do
57
+ # TODO Use a more generic environment variable name if/when
58
+ # Mongoid tests get Kerberos configurations.
59
+ unless %w(1 yes true).include?(ENV['MONGO_RUBY_DRIVER_KERBEROS']&.downcase)
60
+ skip 'Set MONGO_RUBY_DRIVER_KERBEROS=1 in environment to run Kerberos unit tests'
61
+ end
62
+ require 'mongo_kerberos'
63
+ end
64
+ end
65
+
66
+ def require_linting
67
+ before(:all) do
68
+ unless Mongo::Lint.enabled?
69
+ skip "Linting is not enabled"
70
+ end
71
+ end
72
+ end
73
+
74
+ # Some tests will fail if linting is enabled:
75
+ # 1. Tests that pass invalid options to client, etc. which the linter
76
+ # rejects.
77
+ # 2. Tests that set expectations on topologies, server descriptions, etc.
78
+ # (since setting expectations requires mutating said objects, and when
79
+ # linting is on those objects are frozen).
80
+ def require_no_linting
81
+ before(:all) do
82
+ if Mongo::Lint.enabled?
83
+ skip "Linting is enabled"
84
+ end
85
+ end
86
+ end
87
+
88
+ def require_libmongocrypt
89
+ before(:all) do
90
+ unless ENV['LIBMONGOCRYPT_PATH']
91
+ skip 'Test requires path to libmongocrypt to be specified in LIBMONGOCRYPT_PATH env variable'
92
+ end
93
+ end
94
+ end
95
+
96
+ def require_no_libmongocrypt
97
+ before(:all) do
98
+ if ENV['LIBMONGOCRYPT_PATH']
99
+ skip 'Test requires libmongocrypt to not be configured'
100
+ end
101
+ end
102
+ end
103
+
104
+ def require_aws_auth
105
+ before(:all) do
106
+ unless (ENV['AUTH'] || '') =~ /^aws/
107
+ skip 'This test requires AUTH=aws* and an appropriately configured runtime environment'
108
+ end
109
+ end
110
+ end
111
+
112
+ def require_ec2_host
113
+ before(:all) do
114
+ if $have_aws.nil?
115
+ $have_aws = begin
116
+ require 'open-uri'
117
+ begin
118
+ Timeout.timeout(3.81) do
119
+ URI.parse('http://169.254.169.254/latest/meta-data/profile').open.read
120
+ end
121
+ true
122
+ # When trying to use the EC2 metadata endpoint on ECS:
123
+ # Errno::EINVAL: Failed to open TCP connection to 169.254.169.254:80 (Invalid argument - connect(2) for "169.254.169.254" port 80)
124
+ rescue Timeout::Error, Errno::ETIMEDOUT, Errno::EINVAL, OpenURI::HTTPError => $aws_error
125
+ false
126
+ end
127
+ end
128
+ end
129
+ unless $have_aws
130
+ skip "EC2 instance metadata is not available - assuming not running on an EC2 instance: #{$aws_error.class}: #{$aws_error}"
131
+ end
132
+ end
133
+ end
134
+
135
+ def require_stress
136
+ before(:all) do
137
+ if !SpecConfig.instance.stress?
138
+ skip 'Set STRESS=1 in environment to run stress tests'
139
+ end
140
+ end
141
+ end
142
+
143
+ def require_fork
144
+ before(:all) do
145
+ if !SpecConfig.instance.fork?
146
+ skip 'Set FORK=1 in environment to run fork tests'
147
+ end
148
+ end
149
+ end
150
+
151
+ def require_ocsp
152
+ before(:all) do
153
+ if !SpecConfig.instance.ocsp?
154
+ skip 'Set OCSP=1 in environment to run OCSP tests'
155
+ end
156
+ end
157
+ end
158
+
159
+ def require_ocsp_verifier
160
+ before(:all) do
161
+ if !SpecConfig.instance.ocsp_verifier?
162
+ skip 'Set OCSP_VERIFIER=1 in environment to run OCSP verifier tests'
163
+ end
164
+ end
165
+ end
166
+
167
+ def require_ocsp_connectivity
168
+ before(:all) do
169
+ if !SpecConfig.instance.ocsp_connectivity?
170
+ skip 'Set OCSP_CONNECTIVITY=pass or OCSP_CONNECTIVITY=fail in environment to run OCSP connectivity tests'
171
+ end
172
+ end
173
+ end
174
+
175
+ def require_active_support
176
+ before(:all) do
177
+ if !SpecConfig.instance.active_support?
178
+ skip 'This test requires ActiveSupport; set WITH_ACTIVE_SUPPORT=1 in environment'
179
+ end
180
+ end
181
+ end
182
+
183
+ def no_active_support
184
+ before(:all) do
185
+ if SpecConfig.instance.active_support?
186
+ skip 'This test requires no ActiveSupport; unset WITH_ACTIVE_SUPPORT in environment'
187
+ end
188
+ end
189
+ end
190
+ end
191
+ end
@@ -0,0 +1,115 @@
1
+ # frozen_string_literal: true
2
+ # encoding: utf-8
3
+
4
+ autoload :JSON, 'json'
5
+ require 'open-uri'
6
+
7
+ module Mrss
8
+ class ServerVersionRegistry
9
+ class Error < StandardError
10
+ end
11
+
12
+ class UnknownVersion < Error
13
+ end
14
+
15
+ class MissingDownloadUrl < Error
16
+ end
17
+
18
+ class BrokenDownloadUrl < Error
19
+ end
20
+
21
+ def initialize(desired_version, arch)
22
+ @desired_version, @arch = desired_version, arch
23
+ end
24
+
25
+ attr_reader :desired_version, :arch
26
+
27
+ def download_url
28
+ @download_url ||= begin
29
+ version, version_ok = detect_version(current_catalog)
30
+ if version.nil?
31
+ version, full_version_ok = detect_version(full_catalog)
32
+ version_ok ||= full_version_ok
33
+ end
34
+ if version.nil?
35
+ if version_ok
36
+ raise MissingDownloadUrl, "No downloads for version #{desired_version}"
37
+ else
38
+ raise UnknownVersion, "No version #{desired_version}"
39
+ end
40
+ end
41
+ dl = version['downloads'].detect do |dl|
42
+ dl['archive']['url'].index("enterprise-#{arch}") &&
43
+ dl['arch'] == 'x86_64'
44
+ end
45
+ unless dl
46
+ raise MissingDownloadUrl, "No download for #{arch} for #{version['version']}"
47
+ end
48
+ url = dl['archive']['url']
49
+ end
50
+ rescue MissingDownloadUrl
51
+ if %w(4.7 4.7.0).include?(desired_version)
52
+ # 4.7.0 has no advertised downloads but it is downloadable and
53
+ # we do need it. Dirty hack below.
54
+ registry = self.class.new('4.4.3', arch)
55
+ registry.download_url.sub('4.4.3', '4.7.0').tap do |url|
56
+ # Sanity check - ensure the URL we hacked up is a valid one
57
+ io = uri_open(url)
58
+ begin
59
+ io.read(1)
60
+ ensure
61
+ io.close
62
+ end
63
+ end
64
+ else
65
+ raise
66
+ end
67
+ end
68
+
69
+ private
70
+
71
+ def uri_open(*args)
72
+ if RUBY_VERSION < '2.5'
73
+ open(*args)
74
+ else
75
+ URI.open(*args)
76
+ end
77
+ end
78
+
79
+ def detect_version(catalog)
80
+ candidate_versions = catalog['versions'].select do |version|
81
+ version['version'].start_with?(desired_version) &&
82
+ !version['version'].include?('-')
83
+ end
84
+ version_ok = !candidate_versions.empty?
85
+ # Sometimes the download situation is borked and there is a release
86
+ # with no downloads... skip those.
87
+ version = candidate_versions.detect do |version|
88
+ !version['downloads'].empty?
89
+ end
90
+ # Allow RC releases if there isn't a GA release.
91
+ if version.nil?
92
+ candidate_versions = catalog['versions'].select do |version|
93
+ version['version'].start_with?(desired_version)
94
+ end
95
+ version_ok ||= !candidate_versions.empty?
96
+ version = candidate_versions.detect do |version|
97
+ !version['downloads'].empty?
98
+ end
99
+ end
100
+ [version, version_ok]
101
+ end
102
+
103
+ def current_catalog
104
+ @current_catalog ||= begin
105
+ JSON.load(uri_open('http://downloads.mongodb.org/current.json').read)
106
+ end
107
+ end
108
+
109
+ def full_catalog
110
+ @full_catalog ||= begin
111
+ JSON.load(uri_open('http://downloads.mongodb.org/full.json').read)
112
+ end
113
+ end
114
+ end
115
+ end