mongo 2.19.3 → 2.20.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (178) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +37 -1
  3. data/Rakefile +81 -172
  4. data/lib/mongo/cluster/topology/base.rb +16 -0
  5. data/lib/mongo/cluster.rb +27 -1
  6. data/lib/mongo/collection/view/iterable.rb +1 -0
  7. data/lib/mongo/collection.rb +4 -2
  8. data/lib/mongo/config.rb +2 -2
  9. data/lib/mongo/error/transactions_not_supported.rb +34 -0
  10. data/lib/mongo/error.rb +1 -0
  11. data/lib/mongo/grid/fs_bucket.rb +6 -0
  12. data/lib/mongo/monitoring/event/secure.rb +1 -1
  13. data/lib/mongo/operation/shared/executable.rb +43 -27
  14. data/lib/mongo/operation/shared/response_handling.rb +23 -25
  15. data/lib/mongo/retryable/base_worker.rb +28 -3
  16. data/lib/mongo/retryable/read_worker.rb +16 -14
  17. data/lib/mongo/retryable/write_worker.rb +11 -8
  18. data/lib/mongo/retryable.rb +2 -2
  19. data/lib/mongo/server/app_metadata/environment.rb +64 -9
  20. data/lib/mongo/server/app_metadata.rb +5 -4
  21. data/lib/mongo/server/description/features.rb +1 -0
  22. data/lib/mongo/server/pending_connection.rb +19 -6
  23. data/lib/mongo/server_selector/base.rb +32 -6
  24. data/lib/mongo/session/server_session/dirtyable.rb +52 -0
  25. data/lib/mongo/session/server_session.rb +3 -0
  26. data/lib/mongo/session/session_pool.rb +12 -18
  27. data/lib/mongo/session.rb +32 -0
  28. data/lib/mongo/socket/ssl.rb +22 -1
  29. data/lib/mongo/uri.rb +0 -4
  30. data/lib/mongo/version.rb +1 -5
  31. data/mongo.gemspec +9 -18
  32. data/spec/atlas/atlas_connectivity_spec.rb +4 -4
  33. data/spec/faas/ruby-sam-app/Gemfile +9 -0
  34. data/spec/faas/ruby-sam-app/mongodb/Gemfile +4 -0
  35. data/spec/faas/ruby-sam-app/mongodb/app.rb +149 -0
  36. data/spec/faas/ruby-sam-app/template.yaml +48 -0
  37. data/spec/integration/client_side_encryption/corpus_spec.rb +10 -2
  38. data/spec/integration/client_side_encryption/range_explicit_encryption_prose_spec.rb +3 -0
  39. data/spec/integration/retryable_reads_errors_spec.rb +196 -31
  40. data/spec/integration/retryable_writes_errors_spec.rb +156 -0
  41. data/spec/integration/sdam_error_handling_spec.rb +2 -0
  42. data/spec/lite_spec_helper.rb +0 -10
  43. data/spec/mongo/cluster_spec.rb +36 -0
  44. data/spec/mongo/collection/view/aggregation_spec.rb +6 -1
  45. data/spec/mongo/collection/view/explainable_spec.rb +2 -0
  46. data/spec/mongo/collection_crud_spec.rb +2 -1
  47. data/spec/mongo/operation/insert_spec.rb +1 -1
  48. data/spec/mongo/retryable/write_worker_spec.rb +39 -0
  49. data/spec/mongo/server/app_metadata/environment_spec.rb +135 -0
  50. data/spec/mongo/server/app_metadata_spec.rb +12 -2
  51. data/spec/mongo/server/connection_spec.rb +26 -0
  52. data/spec/mongo/session/session_pool_spec.rb +1 -16
  53. data/spec/mongo/session_transaction_spec.rb +15 -0
  54. data/spec/mongo/uri_spec.rb +0 -9
  55. data/spec/runners/crud/test.rb +0 -8
  56. data/spec/runners/crud.rb +1 -1
  57. data/spec/runners/transactions/test.rb +12 -3
  58. data/spec/runners/unified/assertions.rb +16 -3
  59. data/spec/runners/unified/crud_operations.rb +12 -0
  60. data/spec/runners/unified/support_operations.rb +3 -5
  61. data/spec/runners/unified/test.rb +8 -1
  62. data/spec/spec_tests/data/client_side_encryption/explain.yml +2 -2
  63. data/spec/spec_tests/data/client_side_encryption/fle2v2-BypassQueryAnalysis.yml +1 -0
  64. data/spec/spec_tests/data/client_side_encryption/fle2v2-Compact.yml +1 -0
  65. data/spec/spec_tests/data/client_side_encryption/fle2v2-CreateCollection.yml +1 -0
  66. data/spec/spec_tests/data/client_side_encryption/fle2v2-DecryptExistingData.yml +1 -0
  67. data/spec/spec_tests/data/client_side_encryption/fle2v2-Delete.yml +1 -0
  68. data/spec/spec_tests/data/client_side_encryption/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml +1 -0
  69. data/spec/spec_tests/data/client_side_encryption/fle2v2-EncryptedFields-vs-jsonSchema.yml +1 -0
  70. data/spec/spec_tests/data/client_side_encryption/fle2v2-EncryptedFieldsMap-defaults.yml +1 -0
  71. data/spec/spec_tests/data/client_side_encryption/fle2v2-FindOneAndUpdate.yml +1 -0
  72. data/spec/spec_tests/data/client_side_encryption/fle2v2-InsertFind-Indexed.yml +1 -0
  73. data/spec/spec_tests/data/client_side_encryption/fle2v2-InsertFind-Unindexed.yml +1 -0
  74. data/spec/spec_tests/data/client_side_encryption/fle2v2-MissingKey.yml +1 -0
  75. data/spec/spec_tests/data/client_side_encryption/fle2v2-NoEncryption.yml +1 -0
  76. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Date-Aggregate.yml +1 -0
  77. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Date-Correctness.yml +1 -0
  78. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Date-Delete.yml +1 -0
  79. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Date-FindOneAndUpdate.yml +1 -0
  80. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Date-InsertFind.yml +1 -0
  81. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Date-Update.yml +1 -0
  82. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Decimal-Aggregate.yml +1 -0
  83. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Decimal-Correctness.yml +1 -0
  84. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Decimal-Delete.yml +1 -0
  85. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Decimal-FindOneAndUpdate.yml +1 -0
  86. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Decimal-InsertFind.yml +1 -0
  87. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Decimal-Update.yml +1 -0
  88. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-DecimalPrecision-Aggregate.yml +1 -0
  89. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-DecimalPrecision-Correctness.yml +1 -0
  90. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-DecimalPrecision-Delete.yml +1 -0
  91. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-DecimalPrecision-FindOneAndUpdate.yml +1 -0
  92. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-DecimalPrecision-InsertFind.yml +1 -0
  93. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-DecimalPrecision-Update.yml +1 -0
  94. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Double-Aggregate.yml +1 -0
  95. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Double-Correctness.yml +1 -0
  96. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Double-Delete.yml +1 -0
  97. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Double-FindOneAndUpdate.yml +1 -0
  98. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Double-InsertFind.yml +1 -0
  99. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Double-Update.yml +1 -0
  100. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-DoublePrecision-Aggregate.yml +1 -0
  101. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-DoublePrecision-Correctness.yml +1 -0
  102. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-DoublePrecision-Delete.yml +1 -0
  103. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-DoublePrecision-FindOneAndUpdate.yml +1 -0
  104. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-DoublePrecision-InsertFind.yml +1 -0
  105. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-DoublePrecision-Update.yml +1 -0
  106. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Int-Aggregate.yml +1 -0
  107. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Int-Correctness.yml +1 -0
  108. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Int-Delete.yml +1 -0
  109. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Int-FindOneAndUpdate.yml +1 -0
  110. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Int-InsertFind.yml +1 -0
  111. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Int-Update.yml +1 -0
  112. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Long-Aggregate.yml +1 -0
  113. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Long-Correctness.yml +1 -0
  114. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Long-Delete.yml +1 -0
  115. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Long-FindOneAndUpdate.yml +1 -0
  116. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Long-InsertFind.yml +1 -0
  117. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-Long-Update.yml +1 -0
  118. data/spec/spec_tests/data/client_side_encryption/fle2v2-Range-WrongType.yml +1 -0
  119. data/spec/spec_tests/data/client_side_encryption/fle2v2-Update.yml +1 -0
  120. data/spec/spec_tests/data/client_side_encryption/fle2v2-validatorAndPartialFieldExpression.yml +2 -1
  121. data/spec/spec_tests/data/connection_string/invalid-uris.yml +0 -10
  122. data/spec/spec_tests/data/connection_string/valid-options.yml +13 -0
  123. data/spec/spec_tests/data/crud_unified/aggregate-write-readPreference.yml +2 -0
  124. data/spec/spec_tests/data/crud_unified/db-aggregate-write-readPreference.yml +2 -0
  125. data/spec/spec_tests/data/crud_unified/find-test-all-options.yml +348 -0
  126. data/spec/spec_tests/data/index_management/createSearchIndex.yml +5 -3
  127. data/spec/spec_tests/data/index_management/createSearchIndexes.yml +7 -4
  128. data/spec/spec_tests/data/index_management/dropSearchIndex.yml +2 -1
  129. data/spec/spec_tests/data/index_management/listSearchIndexes.yml +13 -7
  130. data/spec/spec_tests/data/index_management/updateSearchIndex.yml +2 -1
  131. data/spec/spec_tests/data/retryable_writes/unified/bulkWrite-serverErrors.yml +3 -6
  132. data/spec/spec_tests/data/retryable_writes/unified/insertOne-serverErrors.yml +3 -6
  133. data/spec/spec_tests/data/run_command_unified/runCommand.yml +319 -0
  134. data/spec/spec_tests/data/sessions_unified/driver-sessions-dirty-session-errors.yml +351 -0
  135. data/spec/spec_tests/data/unified/valid-pass/poc-crud.yml +1 -1
  136. data/spec/spec_tests/data/unified/valid-pass/poc-retryable-writes.yml +7 -7
  137. data/spec/spec_tests/data/unified/valid-pass/poc-sessions.yml +3 -4
  138. data/spec/spec_tests/data/unified/valid-pass/poc-transactions-convenient-api.yml +1 -1
  139. data/spec/spec_tests/data/unified/valid-pass/poc-transactions-mongos-pin-auto.yml +1 -1
  140. data/spec/spec_tests/data/unified/valid-pass/poc-transactions.yml +3 -3
  141. data/spec/spec_tests/run_command_unified_spec.rb +13 -0
  142. data/spec/spec_tests/sdam_unified_spec.rb +2 -0
  143. data/spec/spec_tests/transactions_unified_spec.rb +2 -1
  144. data/spec/support/certificates/atlas-ocsp-ca.crt +89 -77
  145. data/spec/support/certificates/atlas-ocsp.crt +117 -122
  146. data/spec/support/certificates/retrieve-atlas-cert +1 -1
  147. data/spec/support/constraints.rb +6 -0
  148. data/spec/support/ocsp +1 -1
  149. data/spec/support/recording_logger.rb +27 -0
  150. metadata +1245 -1298
  151. checksums.yaml.gz.sig +0 -0
  152. data/spec/shared/LICENSE +0 -20
  153. data/spec/shared/bin/get-mongodb-download-url +0 -17
  154. data/spec/shared/bin/s3-copy +0 -45
  155. data/spec/shared/bin/s3-upload +0 -69
  156. data/spec/shared/lib/mrss/child_process_helper.rb +0 -80
  157. data/spec/shared/lib/mrss/cluster_config.rb +0 -231
  158. data/spec/shared/lib/mrss/constraints.rb +0 -378
  159. data/spec/shared/lib/mrss/docker_runner.rb +0 -295
  160. data/spec/shared/lib/mrss/eg_config_utils.rb +0 -51
  161. data/spec/shared/lib/mrss/event_subscriber.rb +0 -210
  162. data/spec/shared/lib/mrss/lite_constraints.rb +0 -238
  163. data/spec/shared/lib/mrss/server_version_registry.rb +0 -113
  164. data/spec/shared/lib/mrss/session_registry.rb +0 -69
  165. data/spec/shared/lib/mrss/session_registry_legacy.rb +0 -60
  166. data/spec/shared/lib/mrss/spec_organizer.rb +0 -179
  167. data/spec/shared/lib/mrss/utils.rb +0 -37
  168. data/spec/shared/share/Dockerfile.erb +0 -330
  169. data/spec/shared/share/haproxy-1.conf +0 -16
  170. data/spec/shared/share/haproxy-2.conf +0 -17
  171. data/spec/shared/shlib/config.sh +0 -27
  172. data/spec/shared/shlib/distro.sh +0 -74
  173. data/spec/shared/shlib/server.sh +0 -416
  174. data/spec/shared/shlib/set_env.sh +0 -169
  175. data/spec/spec_tests/data/cmap/pool-clear-interrupt-immediately.yml +0 -49
  176. data/spec/support/faas/app/aws_lambda/mongodb/Gemfile.lock +0 -19
  177. data.tar.gz.sig +0 -3
  178. metadata.gz.sig +0 -0
@@ -1,74 +0,0 @@
1
- detected_distro=
2
-
3
- host_distro() {
4
- if test -z "$detected_distro"; then
5
- detected_distro=`_detect_distro`
6
- fi
7
- echo "$detected_distro"
8
- }
9
-
10
- _detect_distro() {
11
- local distro
12
- distro=
13
- if test -f /etc/debian_version; then
14
- # Debian or Ubuntu
15
- if test "`uname -m`" = aarch64; then
16
- release=`lsb_release -rs |tr -d .`
17
- distro="ubuntu$release"-arm
18
- elif lsb_release -is |grep -q Debian; then
19
- release=`lsb_release -rs |tr -d .`
20
- # In docker, release is something like 9.11.
21
- # In evergreen, release is 9.2.
22
- release=`echo $release |sed -e 's/^9.*/92/'`
23
- distro="debian$release"
24
- elif lsb_release -is |grep -q Ubuntu; then
25
- if test "`uname -m`" = ppc64le; then
26
- release=`lsb_release -rs |tr -d .`
27
- distro="ubuntu$release-ppc"
28
- else
29
- release=`lsb_release -rs |tr -d .`
30
- distro="ubuntu$release"
31
- fi
32
- else
33
- echo 'Unknown Debian flavor' 1>&2
34
- exit 1
35
- fi
36
- elif lsb_release -is |grep -qi suse; then
37
- if test "`uname -m`" = s390x; then
38
- release=`lsb_release -rs |sed -e 's/\..*//'`
39
- distro="suse$release-s390x"
40
- else
41
- echo 'Unknown Suse arch' 1>&2
42
- exit 1
43
- fi
44
- elif test -f /etc/redhat-release; then
45
- # RHEL or CentOS
46
- if test "`uname -m`" = s390x; then
47
- distro=rhel72-s390x
48
- elif test "`uname -m`" = ppc64le; then
49
- distro=rhel71-ppc
50
- elif lsb_release >/dev/null 2>&1; then
51
- if lsb_release -is |grep -q RedHat; then
52
- release=`lsb_release -rs |tr -d .`
53
- distro="rhel$release"
54
- elif lsb_release -is |grep -q CentOS; then
55
- release=`lsb_release -rs |cut -c 1 |sed -e s/7/70/ -e s/6/62/ -e s/8/80/`
56
- distro="rhel$release"
57
- else
58
- echo 'Unknown RHEL flavor' 1>&2
59
- exit 1
60
- fi
61
- else
62
- echo lsb_release missing, using /etc/redhat-release 1>&2
63
- release=`grep -o 'release [0-9]' /etc/redhat-release |awk '{print $2}'`
64
- release=`echo $release |sed -e s/7/70/ -e s/6/62/ -e s/8/80/`
65
- distro=rhel$release
66
- fi
67
- else
68
- lsb_release -a
69
- echo 'Unknown distro' 1>&2
70
- exit 1
71
- fi
72
- echo "Detected distro: $distro" 1>&2
73
- echo $distro
74
- }
@@ -1,416 +0,0 @@
1
- # This file contains functions pertaining to downloading, starting and
2
- # configuring a MongoDB server.
3
-
4
- # Note that mlaunch is executed with (and therefore installed with) Python 2.
5
- # The reason for this is that in the past, some of the distros we tested on
6
- # had an ancient version of Python 3 that was unusable (e.g. it couldn't
7
- # install anything from PyPI due to outdated TLS/SSL implementation).
8
- # It is likely that all of the current distros we use have a recent enough
9
- # and working Python 3 implementation, such that we could use Python 3 for
10
- # everything.
11
- #
12
- # Note that some distros (e.g. ubuntu2004) do not contain a `python' binary
13
- # at all, thus python2 or python3 must be explicitly specified depending on
14
- # the desired version.
15
-
16
- set_fcv() {
17
- if test -n "$FCV"; then
18
- mongo --eval 'assert.commandWorked(db.adminCommand( { setFeatureCompatibilityVersion: "'"$FCV"'" } ));' "$MONGODB_URI"
19
- mongo --quiet --eval 'db.adminCommand( { getParameter: 1, featureCompatibilityVersion: 1 } )' |grep "version.*$FCV"
20
- fi
21
- }
22
-
23
- add_uri_option() {
24
- opt=$1
25
-
26
- if ! echo $MONGODB_URI |sed -e s,//,, |grep -q /; then
27
- MONGODB_URI="$MONGODB_URI/"
28
- fi
29
-
30
- if ! echo $MONGODB_URI |grep -q '?'; then
31
- MONGODB_URI="$MONGODB_URI?"
32
- fi
33
-
34
- MONGODB_URI="$MONGODB_URI&$opt"
35
- }
36
-
37
- prepare_server() {
38
- arch=$1
39
-
40
- if test -n "$USE_OPT_MONGODB"; then
41
- export BINDIR=/opt/mongodb/bin
42
- export PATH=$BINDIR:$PATH
43
- return
44
- fi
45
-
46
- if test "$MONGODB_VERSION" = latest; then
47
- # Test on the most recent published 4.3 release.
48
- # https://jira.mongodb.org/browse/RUBY-1724
49
-
50
- . $PROJECT_DIRECTORY/.mod/drivers-evergreen-tools/.evergreen/download-mongodb.sh
51
-
52
- get_distro
53
- get_mongodb_download_url_for "$DISTRO" "latest"
54
- prepare_server_from_url $MONGODB_DOWNLOAD_URL
55
- else
56
- download_version="$MONGODB_VERSION"
57
- url=`$(dirname $0)/get-mongodb-download-url $download_version $arch`
58
- prepare_server_from_url $url
59
- fi
60
-
61
- }
62
-
63
- prepare_server_from_url() {
64
- url=$1
65
-
66
- dirname=`basename $url |sed -e s/.tgz//`
67
- mongodb_dir="$MONGO_ORCHESTRATION_HOME"/mdb/"$dirname"
68
- mkdir -p "$mongodb_dir"
69
- curl --retry 3 $url | tar xz -C "$mongodb_dir" --strip-components 1 -f -
70
- BINDIR="$mongodb_dir"/bin
71
- export PATH="$BINDIR":$PATH
72
- }
73
-
74
- install_mlaunch_venv() {
75
- python3 -V || true
76
- if ! python3 -m venv -h >/dev/null; then
77
- # Current virtualenv fails with
78
- # https://github.com/pypa/virtualenv/issues/1630
79
- python3 -m pip install venv --user
80
- fi
81
- if ! python3 -m ensurepip -h > /dev/null; then
82
- # Debian11/Ubuntu2204 have venv installed, but it is nonfunctional unless
83
- # the python3-venv package is also installed (it lacks the ensurepip
84
- # module).
85
- sudo apt-get install --yes python3-venv
86
- fi
87
- if test "$USE_SYSTEM_PYTHON_PACKAGES" = 1 &&
88
- python3 -m pip list |grep mtools
89
- then
90
- # Use the existing mtools-legacy
91
- :
92
- else
93
- # Spawn a virtual environment, but only if one is not already
94
- # active...
95
- if test -z "$VIRTUAL_ENV"; then
96
- venvpath="$MONGO_ORCHESTRATION_HOME"/venv
97
- python3 -m venv $venvpath
98
- . $venvpath/bin/activate
99
- fi
100
-
101
- # [mlaunch] does not work:
102
- # https://github.com/rueckstiess/mtools/issues/856
103
- # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
104
- #pip install 'mtools==1.7' 'pymongo==4.1' python-dateutil psutil
105
-
106
- # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
107
- pip install 'mtools-legacy[mlaunch]' 'pymongo<4' python-dateutil
108
- fi
109
- }
110
-
111
- install_mlaunch_pip() {
112
- if test -n "$USE_OPT_MONGODB" && which mlaunch >/dev/null 2>&1; then
113
- # mlaunch is preinstalled in the docker image, do not install it here
114
- return
115
- fi
116
-
117
- python -V || true
118
- python3 -V || true
119
- pythonpath="$MONGO_ORCHESTRATION_HOME"/python
120
- # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
121
- pip install -t "$pythonpath" 'mtools-legacy[mlaunch]' 'pymongo<4' python-dateutil
122
- export PATH="$pythonpath/bin":$PATH
123
- export PYTHONPATH="$pythonpath"
124
- }
125
-
126
- install_mlaunch_git() {
127
- repo=$1
128
- branch=$2
129
- python -V || true
130
- python3 -V || true
131
- which pip || true
132
- which pip3 || true
133
-
134
- if false; then
135
- if ! virtualenv --version; then
136
- python3 `which pip3` install --user virtualenv
137
- export PATH=$HOME/.local/bin:$PATH
138
- virtualenv --version
139
- fi
140
-
141
- venvpath="$MONGO_ORCHESTRATION_HOME"/venv
142
- virtualenv -p python3 $venvpath
143
- . $venvpath/bin/activate
144
-
145
- # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
146
- pip3 install psutil pymongo python-dateutil
147
-
148
- git clone $repo mlaunch
149
- cd mlaunch
150
- git checkout origin/$branch
151
- python3 setup.py install
152
- cd ..
153
- else
154
- pip install --user 'virtualenv==13'
155
- export PATH=$HOME/.local/bin:$PATH
156
-
157
- venvpath="$MONGO_ORCHESTRATION_HOME"/venv
158
- virtualenv $venvpath
159
- . $venvpath/bin/activate
160
-
161
- # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
162
- pip install psutil pymongo python-dateutil
163
-
164
- git clone $repo mlaunch
165
- (cd mlaunch &&
166
- git checkout origin/$branch &&
167
- python2 setup.py install
168
- )
169
- fi
170
- }
171
-
172
- install_cmake() {
173
- if ! command -v cmake &> /dev/null; then
174
- if ! command -v apt-get &> /dev/null; then
175
- # no apt-get; assume RHEL
176
- sudo yum -y install cmake libarchive
177
- else
178
- sudo apt-get install --yes cmake
179
- fi
180
- else
181
- echo 'cmake is present'
182
- fi
183
- }
184
-
185
- # This function sets followong global variables:
186
- # server_cert_path
187
- # server_ca_path
188
- # server_client_cert_path
189
- #
190
- # These variables are used later to connect to processes via mongo client.
191
- calculate_server_args() {
192
- local mongo_version=`echo $MONGODB_VERSION |tr -d .`
193
-
194
- if test -z "$mongo_version"; then
195
- echo "$MONGODB_VERSION must be set and not contain only dots" 1>&2
196
- exit 3
197
- fi
198
-
199
- if test $mongo_version = latest; then
200
- mongo_version=70
201
- fi
202
-
203
- local args="--setParameter enableTestCommands=1"
204
-
205
- if test $mongo_version -ge 50; then
206
- args="$args --setParameter acceptApiVersion2=1"
207
- elif test $mongo_version -ge 47; then
208
- args="$args --setParameter acceptAPIVersion2=1"
209
- fi
210
-
211
- # diagnosticDataCollectionEnabled is a mongod-only parameter on server 3.2,
212
- # and mlaunch does not support specifying mongod-only parameters:
213
- # https://github.com/rueckstiess/mtools/issues/696
214
- # Pass it to 3.4 and newer servers where it is accepted by all daemons.
215
- if test $mongo_version -ge 34; then
216
- args="$args --setParameter diagnosticDataCollectionEnabled=false"
217
- fi
218
- local uri_options=
219
- if test "$TOPOLOGY" = replica-set; then
220
- args="$args --replicaset --name test-rs --nodes 2 --arbiter"
221
- export HAVE_ARBITER=1
222
- elif test "$TOPOLOGY" = sharded-cluster; then
223
- args="$args --replicaset --nodes 2 --sharded 1 --name test-rs"
224
- if test -z "$SINGLE_MONGOS"; then
225
- args="$args --mongos 2"
226
- fi
227
- elif test "$TOPOLOGY" = standalone; then
228
- args="$args --single"
229
- elif test "$TOPOLOGY" = load-balanced; then
230
- args="$args --replicaset --nodes 2 --sharded 1 --name test-rs --port 27117"
231
- if test -z "$MRSS_ROOT"; then
232
- echo "Please set MRSS_ROOT" 1>&2
233
- exit 2
234
- fi
235
- if test -n "$SINGLE_MONGOS"; then
236
- haproxy_config=$MRSS_ROOT/share/haproxy-1.conf
237
- else
238
- args="$args --mongos 2"
239
- haproxy_config=$MRSS_ROOT/share/haproxy-2.conf
240
- fi
241
- uri_options="$uri_options&loadBalanced=true"
242
- else
243
- echo "Unknown topology: $TOPOLOGY" 1>&2
244
- exit 1
245
- fi
246
- if test -n "$MMAPV1"; then
247
- args="$args --storageEngine mmapv1 --smallfiles --noprealloc"
248
- uri_options="$uri_options&retryReads=false&retryWrites=false"
249
- fi
250
- if test "$AUTH" = auth; then
251
- args="$args --auth --username bob --password pwd123"
252
- elif test "$AUTH" = x509; then
253
- args="$args --auth --username bootstrap --password bootstrap"
254
- elif echo "$AUTH" |grep -q ^aws; then
255
- args="$args --auth --username bootstrap --password bootstrap"
256
- args="$args --setParameter authenticationMechanisms=MONGODB-AWS,SCRAM-SHA-1,SCRAM-SHA-256"
257
- uri_options="$uri_options&authMechanism=MONGODB-AWS&authSource=\$external"
258
- fi
259
-
260
- if test -n "$OCSP"; then
261
- if test -z "$OCSP_ALGORITHM"; then
262
- echo "OCSP_ALGORITHM must be set if OCSP is set" 1>&2
263
- exit 1
264
- fi
265
- fi
266
-
267
- if test "$SSL" = ssl || test -n "$OCSP_ALGORITHM"; then
268
- if test -n "$OCSP_ALGORITHM"; then
269
- if test "$OCSP_MUST_STAPLE" = 1; then
270
- server_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server-mustStaple.pem
271
- else
272
- server_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server.pem
273
- fi
274
- server_ca_path=spec/support/ocsp/$OCSP_ALGORITHM/ca.crt
275
- server_client_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server.pem
276
- else
277
- server_cert_path=spec/support/certificates/server-second-level-bundle.pem
278
- server_ca_path=spec/support/certificates/ca.crt
279
- server_client_cert_path=spec/support/certificates/client.pem
280
- fi
281
-
282
- if test -n "$OCSP_ALGORITHM"; then
283
- client_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server.pem
284
- elif test "$AUTH" = x509; then
285
- client_cert_path=spec/support/certificates/client-x509.pem
286
-
287
- uri_options="$uri_options&authMechanism=MONGODB-X509"
288
- elif echo $RVM_RUBY |grep -q jruby; then
289
- # JRuby does not grok chained certificate bundles -
290
- # https://github.com/jruby/jruby-openssl/issues/181
291
- client_cert_path=spec/support/certificates/client.pem
292
- else
293
- client_cert_path=spec/support/certificates/client-second-level-bundle.pem
294
- fi
295
-
296
- uri_options="$uri_options&tls=true&"\
297
- "tlsCAFile=$server_ca_path&"\
298
- "tlsCertificateKeyFile=$client_cert_path"
299
-
300
- args="$args --sslMode requireSSL"\
301
- " --sslPEMKeyFile $server_cert_path"\
302
- " --sslCAFile $server_ca_path"\
303
- " --sslClientCertificate $server_client_cert_path"
304
- fi
305
-
306
- # Docker forwards ports to the external interface, not to the loopback.
307
- # Hence we must bind to all interfaces here.
308
- if test -n "$BIND_ALL"; then
309
- args="$args --bind_ip_all"
310
- fi
311
-
312
- # MongoDB servers pre-4.2 do not enable zlib compression by default
313
- if test "$COMPRESSOR" = snappy; then
314
- args="$args --networkMessageCompressors snappy"
315
- elif test "$COMPRESSOR" = zlib; then
316
- args="$args --networkMessageCompressors zlib"
317
- fi
318
-
319
- if test -n "$OCSP_ALGORITHM" || test -n "$OCSP_VERIFIER"; then
320
- python3 -m pip install asn1crypto oscrypto flask
321
- fi
322
-
323
- local ocsp_args=
324
- if test -n "$OCSP_ALGORITHM"; then
325
- if test -z "$server_ca_path"; then
326
- echo "server_ca_path must have been set" 1>&2
327
- exit 1
328
- fi
329
- ocsp_args="--ca_file $server_ca_path"
330
- if test "$OCSP_DELEGATE" = 1; then
331
- ocsp_args="$ocsp_args \
332
- --ocsp_responder_cert spec/support/ocsp/$OCSP_ALGORITHM/ocsp-responder.crt \
333
- --ocsp_responder_key spec/support/ocsp/$OCSP_ALGORITHM/ocsp-responder.key \
334
- "
335
- else
336
- ocsp_args="$ocsp_args \
337
- --ocsp_responder_cert spec/support/ocsp/$OCSP_ALGORITHM/ca.crt \
338
- --ocsp_responder_key spec/support/ocsp/$OCSP_ALGORITHM/ca.key \
339
- "
340
- fi
341
- if test -n "$OCSP_STATUS"; then
342
- ocsp_args="$ocsp_args --fault $OCSP_STATUS"
343
- fi
344
- fi
345
-
346
- OCSP_ARGS="$ocsp_args"
347
- SERVER_ARGS="$args"
348
- URI_OPTIONS="$uri_options"
349
- }
350
-
351
- launch_ocsp_mock() {
352
- if test -n "$OCSP_ARGS"; then
353
- # Bind to 0.0.0.0 for Docker
354
- python3 spec/support/ocsp/ocsp_mock.py $OCSP_ARGS -b 0.0.0.0 -p 8100 &
355
- OCSP_MOCK_PID=$!
356
- fi
357
- }
358
-
359
- launch_server() {
360
- local dbdir="$1"
361
- python3 -m mtools.mlaunch.mlaunch --dir "$dbdir" --binarypath "$BINDIR" $SERVER_ARGS
362
-
363
- if test "$TOPOLOGY" = sharded-cluster && test $MONGODB_VERSION = 3.6; then
364
- # On 3.6 server the sessions collection is not immediately available,
365
- # so we run the refreshLogicalSessionCacheNow command on the config server
366
- # and again on each mongos in order for the mongoses
367
- # to correctly report logicalSessionTimeoutMinutes.
368
- mongos_regex="\s*mongos\s+([0-9]+)\s+running\s+[0-9]+"
369
- config_server_regex="\s*config\sserver\s+([0-9]+)\s+running\s+[0-9]+"
370
- config_server=""
371
- mongoses=()
372
- if test "$AUTH" = auth
373
- then
374
- base_url="mongodb://bob:pwd123@localhost"
375
- else
376
- base_url="mongodb://localhost"
377
- fi
378
- if test "$SSL" = "ssl"
379
- then
380
- mongo_command="${BINDIR}/mongo --ssl --sslPEMKeyFile $server_cert_path --sslCAFile $server_ca_path"
381
- else
382
- mongo_command="${BINDIR}/mongo"
383
- fi
384
-
385
- while read -r line
386
- do
387
- if [[ $line =~ $config_server_regex ]]
388
- then
389
- port="${BASH_REMATCH[1]}"
390
- config_server="${base_url}:${port}"
391
- fi
392
- if [[ $line =~ $mongos_regex ]]
393
- then
394
- port="${BASH_REMATCH[1]}"
395
- mongoses+=("${base_url}:${port}")
396
- fi
397
- done < <(python2 -m mtools.mlaunch.mlaunch list --dir "$dbdir" --binarypath "$BINDIR")
398
-
399
- if [ -n "$config_server" ]; then
400
- ${mongo_command} "$config_server" --eval 'db.adminCommand("refreshLogicalSessionCacheNow")'
401
- for mongos in ${mongoses[*]}
402
- do
403
- ${mongo_command} "$mongos" --eval 'db.adminCommand("refreshLogicalSessionCacheNow")'
404
- done
405
- fi
406
- fi
407
-
408
- if test "$TOPOLOGY" = load-balanced; then
409
- if test -z "$haproxy_config"; then
410
- echo haproxy_config should have been set 1>&2
411
- exit 3
412
- fi
413
-
414
- haproxy -D -f $haproxy_config -p $mongodb_dir/haproxy.pid
415
- fi
416
- }
@@ -1,169 +0,0 @@
1
- # When changing, also update the hash in share/Dockerfile.
2
- TOOLCHAIN_VERSION=e8c60866f54bed7e336a37df3a97d6ae1b971b7d
3
-
4
- set_env_java() {
5
- ls -l /opt || true
6
- ls -l /usr/lib/jvm || true
7
-
8
- # Use toolchain java if it exists
9
- if [ -f /opt/java/jdk8/bin/java ]; then
10
- export JAVACMD=/opt/java/jdk8/bin/java
11
- #export PATH=$PATH:/opt/java/jdk8/bin
12
- fi
13
-
14
- # ppc64le has it in a different place
15
- if test -z "$JAVACMD" && [ -f /usr/lib/jvm/java-1.8.0/bin/java ]; then
16
- export JAVACMD=/usr/lib/jvm/java-1.8.0/bin/java
17
- #export PATH=$PATH:/usr/lib/jvm/java-1.8.0/bin
18
- fi
19
-
20
- if true; then
21
- # newer
22
- # rhel71-ppc, https://jira.mongodb.org/browse/BUILD-9231
23
- if test -z "$JAVACMD" &&
24
- (ls /opt/java || true) |grep -q java-1.8.0-openjdk-1.8.0 &&
25
- test -f /opt/java/java-1.8.0-openjdk-1.8.0*/bin/java;
26
- then
27
- path=$(cd /opt/java && ls -d java-1.8.0-openjdk-1.8.0* |head -n 1)
28
- export JAVACMD=/opt/java/"$path"/bin/java
29
- fi
30
- else
31
- # older
32
- # rhel71-ppc seems to have an /opt/java/jdk8/bin/java but it doesn't work
33
- if test -n "$JAVACMD" && ! exec $JAVACMD -version; then
34
- JAVACMD=
35
- # we will try the /usr/lib/jvm then
36
- fi
37
- fi
38
-
39
- if test -n "$JAVACMD"; then
40
- eval $JAVACMD -version
41
- elif which java 2>/dev/null; then
42
- java -version
43
- else
44
- echo No java runtime found
45
- fi
46
- }
47
-
48
- set_env_python() {
49
- if test "$DOCKER_PRELOAD" != 1; then
50
- if test -n "$DOCKER"; then
51
- # If we are running in Docker and not preloading, we need to fetch the
52
- # Python binary.
53
- curl -fL --retry 3 https://github.com/p-mongodb/deps/raw/main/"$arch"-python37.tar.xz | \
54
- tar xfJ - -C /opt
55
- fi
56
-
57
- if test -d /opt/python/3.7/bin; then
58
- # Most Evergreen configurations.
59
- export PATH=/opt/python/3.7/bin:$PATH
60
- elif test -d /opt/python37/bin; then
61
- # Configurations that use Docker in Evergreen - these don't preload.
62
- export PATH=/opt/python37/bin:$PATH
63
- fi
64
-
65
- python3 -V
66
- fi
67
- }
68
-
69
- set_env_node() {
70
- if test "$DOCKER_PRELOAD" != 1; then
71
- dir=`ls -d /opt/nodejs/node-v12* |head -1`
72
- if test -z "$dir"; then
73
- echo "Node 12 missing" 1>&2
74
- exit 2
75
- fi
76
- export PATH="$dir/bin:$PATH"
77
- elif test -d /opt/node/bin; then
78
- # Node from toolchain in Evergreen
79
- export PATH=/opt/node/bin:$PATH
80
- fi
81
-
82
- node -v
83
- }
84
-
85
- set_env_ruby() {
86
- if test -z "$RVM_RUBY"; then
87
- echo "Empty RVM_RUBY, aborting"
88
- exit 2
89
- fi
90
-
91
- #ls -l /opt
92
-
93
- # Necessary for jruby
94
- set_env_java
95
-
96
- if [ "$RVM_RUBY" == "ruby-head" ]; then
97
- # When we use ruby-head, we do not install the Ruby toolchain.
98
- # But we still need Python 3.6+ to run mlaunch.
99
- # Since the ruby-head tests are run on ubuntu1604, we can use the
100
- # globally installed Python toolchain.
101
- #export PATH=/opt/python/3.7/bin:$PATH
102
-
103
- # 12.04, 14.04 and 16.04 are good
104
- curl --retry 3 -fL http://rubies.travis-ci.org/ubuntu/`lsb_release -rs`/x86_64/ruby-head.tar.bz2 |tar xfj -
105
- # TODO adjust gem path?
106
- export PATH=`pwd`/ruby-head/bin:`pwd`/ruby-head/lib/ruby/gems/2.6.0/bin:$PATH
107
- ruby --version
108
- ruby --version |grep dev
109
- elif test "$SYSTEM_RUBY" = 1; then
110
- # Nothing
111
- :
112
- else
113
- if test "$USE_OPT_TOOLCHAIN" = 1; then
114
- # Nothing, also PATH is already set
115
- :
116
- elif true; then
117
-
118
- # For testing toolchains:
119
- #toolchain_url=https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/`host_distro`/f11598d091441ffc8d746aacfdc6c26741a3e629/mongo_ruby_driver_toolchain_`host_distro |tr - _`_patch_f11598d091441ffc8d746aacfdc6c26741a3e629_5e46f2793e8e866f36eda2c5_20_02_14_19_18_18.tar.gz
120
- toolchain_url=http://boxes.10gen.com/build/toolchain-drivers/mongo-ruby-driver/$TOOLCHAIN_VERSION/`host_distro`/$RVM_RUBY.tar.xz
121
- curl --retry 3 -fL $toolchain_url |tar Jxf -
122
- export PATH=`pwd`/rubies/$RVM_RUBY/bin:$PATH
123
- #export PATH=`pwd`/rubies/python/3/bin:$PATH
124
-
125
- # Attempt to get bundler to report all errors - so far unsuccessful
126
- #curl --retry 3 -o bundler-openssl.diff https://github.com/bundler/bundler/compare/v2.0.1...p-mongo:report-errors.diff
127
- #find . -path \*/lib/bundler/fetcher.rb -exec patch {} bundler-openssl.diff \;
128
-
129
- else
130
-
131
- # Normal operation
132
- if ! test -d $HOME/.rubies/$RVM_RUBY/bin; then
133
- echo "Ruby directory does not exist: $HOME/.rubies/$RVM_RUBY/bin" 1>&2
134
- echo "Contents of /opt:" 1>&2
135
- ls -l /opt 1>&2 || true
136
- echo ".rubies symlink:" 1>&2
137
- ls -ld $HOME/.rubies 1>&2 || true
138
- echo "Our rubies:" 1>&2
139
- ls -l $HOME/.rubies 1>&2 || true
140
- exit 2
141
- fi
142
- export PATH=$HOME/.rubies/$RVM_RUBY/bin:$PATH
143
-
144
- fi
145
-
146
- ruby --version
147
-
148
- # Ensure we're using the right ruby
149
- ruby_name=`echo $RVM_RUBY |awk -F- '{print $1}'`
150
- ruby_version=`echo $RVM_RUBY |awk -F- '{print $2}' |cut -c 1-3`
151
-
152
- ruby -v |fgrep $ruby_name
153
- ruby -v |fgrep $ruby_version
154
-
155
- # We shouldn't need to update rubygems, and there is value in
156
- # testing on whatever rubygems came with each supported ruby version
157
- #echo 'updating rubygems'
158
- #gem update --system
159
-
160
- # Only install bundler when not using ruby-head.
161
- # ruby-head comes with bundler and gem complains
162
- # because installing bundler would overwrite the bundler binary.
163
- # We now install bundler in the toolchain, hence nothing needs to be done
164
- # in the tests.
165
- if false && echo "$RVM_RUBY" |grep -q jruby; then
166
- gem install bundler -v '<2'
167
- fi
168
- fi
169
- }