bson 4.12.1 → 4.13.0

data/spec/shared/shlib/server.sh

@@ -0,0 +1,290 @@
+# This file contains functions pertaining to downloading, starting and
+# configuring a MongoDB server.
+
+set_fcv() {
+  if test -n "$FCV"; then
+    mongo --eval 'assert.commandWorked(db.adminCommand( { setFeatureCompatibilityVersion: "'"$FCV"'" } ));' "$MONGODB_URI"
+    mongo --quiet --eval 'db.adminCommand( { getParameter: 1, featureCompatibilityVersion: 1 } )' |grep  "version.*$FCV"
+  fi
+}
+
+add_uri_option() {
+  opt=$1
+  
+  if ! echo $MONGODB_URI |sed -e s,//,, |grep -q /; then
+    MONGODB_URI="$MONGODB_URI/"
+  fi
+  
+  if ! echo $MONGODB_URI |grep -q '?'; then
+    MONGODB_URI="$MONGODB_URI?"
+  fi
+  
+  MONGODB_URI="$MONGODB_URI&$opt"
+}
+
+prepare_server() {
+  arch=$1
+  
+  if test -n "$USE_OPT_MONGODB"; then
+    export BINDIR=/opt/mongodb/bin
+    export PATH=$BINDIR:$PATH
+    return
+  fi
+
+  if test "$MONGODB_VERSION" = latest; then
+    # Test on the most recent published 4.3 release.
+    # https://jira.mongodb.org/browse/RUBY-1724
+    echo 'Using "latest" server is not currently implemented' 1>&2
+    exit 1
+  else
+    download_version="$MONGODB_VERSION"
+  fi
+  
+  url=`$(dirname $0)/get-mongodb-download-url $download_version $arch`
+
+  prepare_server_from_url $url
+}
+
+prepare_server_from_url() {
+  url=$1
+
+  mongodb_dir="$MONGO_ORCHESTRATION_HOME"/mdb
+  mkdir -p "$mongodb_dir"
+  curl --retry 3 $url |tar xz -C "$mongodb_dir" -f -
+  BINDIR="$mongodb_dir"/`basename $url |sed -e s/.tgz//`/bin
+  export PATH="$BINDIR":$PATH
+}
+
+install_mlaunch_virtualenv() {
+  python2 -V || true
+  if ! python2 -m virtualenv -h >/dev/null; then
+    # Current virtualenv fails with
+    # https://github.com/pypa/virtualenv/issues/1630
+    python2 -m pip install 'virtualenv<20' --user
+  fi
+  if test "$USE_SYSTEM_PYTHON_PACKAGES" = 1 &&
+    python2 -m pip list |grep mtools-legacy
+  then
+    # Use the existing mtools-legacy
+    :
+  else
+    venvpath="$MONGO_ORCHESTRATION_HOME"/venv
+    python2 -m virtualenv -p python2 $venvpath
+    . $venvpath/bin/activate
+    pip install 'mtools-legacy[mlaunch]'
+  fi
+}
+
+install_mlaunch_pip() {
+  if test -n "$USE_OPT_MONGODB" && which mlaunch >/dev/null 2>&1; then
+    # mlaunch is preinstalled in the docker image, do not install it here
+    return
+  fi
+  
+  python -V || true
+  python3 -V || true
+  pythonpath="$MONGO_ORCHESTRATION_HOME"/python
+  pip install -t "$pythonpath" 'mtools-legacy[mlaunch]'
+  export PATH="$pythonpath/bin":$PATH
+  export PYTHONPATH="$pythonpath"
+}
+
+install_mlaunch_git() {
+  repo=$1
+  branch=$2
+  python -V || true
+  python3 -V || true
+  which pip || true
+  which pip3 || true
+  
+  if false; then
+    if ! virtualenv --version; then
+      python3 `which pip3` install --user virtualenv
+      export PATH=$HOME/.local/bin:$PATH
+      virtualenv --version
+    fi
+    
+    venvpath="$MONGO_ORCHESTRATION_HOME"/venv
+    virtualenv -p python3 $venvpath
+    . $venvpath/bin/activate
+    
+    pip3 install psutil pymongo
+    
+    git clone $repo mlaunch
+    cd mlaunch
+    git checkout origin/$branch
+    python3 setup.py install
+    cd ..
+  else
+    pip install --user 'virtualenv==13'
+    export PATH=$HOME/.local/bin:$PATH
+    
+    venvpath="$MONGO_ORCHESTRATION_HOME"/venv
+    virtualenv $venvpath
+    . $venvpath/bin/activate
+  
+    pip install psutil pymongo
+    
+    git clone $repo mlaunch
+    (cd mlaunch &&
+      git checkout origin/$branch &&
+      python setup.py install
+    )
+  fi
+}
+
+calculate_server_args() {
+  local mongo_version=`echo $MONGODB_VERSION |tr -d .`
+  
+  if test -z "$mongo_version"; then
+    echo "$MONGODB_VERSION must be set and not contain only dots" 1>&2
+    exit 3
+  fi
+  
+  if test $mongo_version = latest; then
+    mongo_version=49
+  fi
+
+  local args="--setParameter enableTestCommands=1"
+  
+  if test $mongo_version -ge 47; then
+    args="$args --setParameter acceptAPIVersion2=1"
+  fi
+  
+  # diagnosticDataCollectionEnabled is a mongod-only parameter on server 3.2,
+  # and mlaunch does not support specifying mongod-only parameters:
+  # https://github.com/rueckstiess/mtools/issues/696
+  # Pass it to 3.4 and newer servers where it is accepted by all daemons.
+  if test $mongo_version -ge 34; then
+    args="$args --setParameter diagnosticDataCollectionEnabled=false"
+  fi
+  local uri_options=
+  if test "$TOPOLOGY" = replica-set; then
+    args="$args --replicaset --name ruby-driver-rs --nodes 2 --arbiter"
+    export HAVE_ARBITER=1
+  elif test "$TOPOLOGY" = sharded-cluster; then
+    args="$args --replicaset --nodes 2 --sharded 1 --name ruby-driver-rs"
+    if test -z "$SINGLE_MONGOS"; then
+      args="$args --mongos 2"
+    fi
+  else
+    args="$args --single"
+  fi
+  if test -n "$MMAPV1"; then
+    args="$args --storageEngine mmapv1 --smallfiles --noprealloc"
+    uri_options="$uri_options&retryReads=false&retryWrites=false"
+  fi
+  if test "$AUTH" = auth; then
+    args="$args --auth --username bob --password pwd123"
+  elif test "$AUTH" = x509; then
+    args="$args --auth --username bootstrap --password bootstrap"
+  elif echo "$AUTH" |grep -q ^aws; then
+    args="$args --auth --username bootstrap --password bootstrap"
+    args="$args --setParameter authenticationMechanisms=MONGODB-AWS,SCRAM-SHA-1,SCRAM-SHA-256"
+    uri_options="$uri_options&authMechanism=MONGODB-AWS&authSource=\$external"
+  fi
+
+  if test -n "$OCSP"; then
+    if test -z "$OCSP_ALGORITHM"; then
+      echo "OCSP_ALGORITHM must be set if OCSP is set" 1>&2
+      exit 1
+    fi
+  fi
+
+  local server_cert_path server_ca_path server_client_cert_path
+  if test "$SSL" = ssl || test -n "$OCSP_ALGORITHM"; then
+    if test -n "$OCSP_ALGORITHM"; then
+      if test "$OCSP_MUST_STAPLE" = 1; then
+        server_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server-mustStaple.pem
+      else
+        server_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server.pem
+      fi
+      server_ca_path=spec/support/ocsp/$OCSP_ALGORITHM/ca.crt
+      server_client_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server.pem
+    else
+      server_cert_path=spec/support/certificates/server-second-level-bundle.pem
+      server_ca_path=spec/support/certificates/ca.crt
+      server_client_cert_path=spec/support/certificates/client.pem
+    fi
+
+    if test -n "$OCSP_ALGORITHM"; then
+      client_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server.pem
+    elif test "$AUTH" = x509; then
+      client_cert_path=spec/support/certificates/client-x509.pem
+
+      uri_options="$uri_options&authMechanism=MONGODB-X509"
+    elif echo $RVM_RUBY |grep -q jruby; then
+      # JRuby does not grok chained certificate bundles -
+      # https://github.com/jruby/jruby-openssl/issues/181
+      client_cert_path=spec/support/certificates/client.pem
+    else
+      client_cert_path=spec/support/certificates/client-second-level-bundle.pem
+    fi
+
+    uri_options="$uri_options&tls=true&"\
+"tlsCAFile=$server_ca_path&"\
+"tlsCertificateKeyFile=$client_cert_path"
+
+    args="$args --sslMode requireSSL"\
+" --sslPEMKeyFile $server_cert_path"\
+" --sslCAFile $server_ca_path"\
+" --sslClientCertificate $server_client_cert_path"
+  fi
+
+  # Docker forwards ports to the external interface, not to the loopback.
+  # Hence we must bind to all interfaces here.
+  if test -n "$BIND_ALL"; then
+    args="$args --bind_ip_all"
+  fi
+
+  # MongoDB servers pre-4.2 do not enable zlib compression by default
+  if test "$COMPRESSOR" = snappy; then
+    args="$args --networkMessageCompressors snappy"
+  elif test "$COMPRESSOR" = zlib; then
+    args="$args --networkMessageCompressors zlib"
+  fi
+
+  if test -n "$OCSP_ALGORITHM" || test -n "$OCSP_VERIFIER"; then
+    python3 -m pip install asn1crypto oscrypto flask
+  fi
+
+  local ocsp_args=
+  if test -n "$OCSP_ALGORITHM"; then
+    if test -z "$server_ca_path"; then
+      echo "server_ca_path must have been set" 1>&2
+      exit 1
+    fi
+    ocsp_args="--ca_file $server_ca_path"
+    if test "$OCSP_DELEGATE" = 1; then
+      ocsp_args="$ocsp_args \
+  --ocsp_responder_cert spec/support/ocsp/$OCSP_ALGORITHM/ocsp-responder.crt \
+  --ocsp_responder_key spec/support/ocsp/$OCSP_ALGORITHM/ocsp-responder.key \
+  "
+    else
+      ocsp_args="$ocsp_args \
+  --ocsp_responder_cert spec/support/ocsp/$OCSP_ALGORITHM/ca.crt \
+  --ocsp_responder_key spec/support/ocsp/$OCSP_ALGORITHM/ca.key \
+  "
+    fi
+    if test -n "$OCSP_STATUS"; then
+      ocsp_args="$ocsp_args --fault $OCSP_STATUS"
+    fi
+  fi
+  
+  OCSP_ARGS="$ocsp_args"
+  SERVER_ARGS="$args"
+  URI_OPTIONS="$uri_options"
+}
+
+launch_ocsp_mock() {
+  if test -n "$OCSP_ARGS"; then
+    # Bind to 0.0.0.0 for Docker
+    python3 spec/support/ocsp/ocsp_mock.py $OCSP_ARGS -b 0.0.0.0 -p 8100 &
+    OCSP_MOCK_PID=$!
+  fi
+}
+
+launch_server() {
+  local dbdir="$1"
+  python -m mtools.mlaunch.mlaunch --dir "$dbdir" --binarypath "$BINDIR" $SERVER_ARGS
+}

data/spec/shared/shlib/set_env.sh

@@ -0,0 +1,128 @@
+TOOLCHAIN_VERSION=289d4bec7c61e88000cea582c05fd8073b932122
+
+set_env_java() {
+  ls -l /opt || true
+  ls -l /usr/lib/jvm || true
+
+  # Use toolchain java if it exists
+  if [ -f /opt/java/jdk8/bin/java ]; then
+    export JAVACMD=/opt/java/jdk8/bin/java
+    #export PATH=$PATH:/opt/java/jdk8/bin
+  fi
+  
+  # ppc64le has it in a different place
+  if test -z "$JAVACMD" && [ -f /usr/lib/jvm/java-1.8.0/bin/java ]; then
+    export JAVACMD=/usr/lib/jvm/java-1.8.0/bin/java
+    #export PATH=$PATH:/usr/lib/jvm/java-1.8.0/bin
+  fi
+  
+  if true; then
+    # newer
+    # rhel71-ppc, https://jira.mongodb.org/browse/BUILD-9231
+    if test -z "$JAVACMD" &&
+      (ls /opt/java || true) |grep -q java-1.8.0-openjdk-1.8.0 &&
+      test -f /opt/java/java-1.8.0-openjdk-1.8.0*/bin/java;
+    then
+      path=$(cd /opt/java && ls -d java-1.8.0-openjdk-1.8.0* |head -n 1)
+      export JAVACMD=/opt/java/"$path"/bin/java
+    fi
+  else
+    # older
+    # rhel71-ppc seems to have an /opt/java/jdk8/bin/java but it doesn't work
+    if test -n "$JAVACMD" && ! exec $JAVACMD -version; then
+      JAVACMD=
+      # we will try the /usr/lib/jvm then
+    fi
+  fi
+  
+  if test -n "$JAVACMD"; then
+    eval $JAVACMD -version
+  elif which java 2>/dev/null; then
+    java -version
+  else
+    echo No java runtime found
+  fi
+}
+
+set_env_ruby() {
+  if test -z "$RVM_RUBY"; then
+    echo "Empty RVM_RUBY, aborting"
+    exit 2
+  fi
+
+  #ls -l /opt
+
+  # Necessary for jruby
+  set_env_java
+
+  if [ "$RVM_RUBY" == "ruby-head" ]; then
+    # When we use ruby-head, we do not install the Ruby toolchain.
+    # But we still need Python 3.6+ to run mlaunch.
+    # Since the ruby-head tests are run on ubuntu1604, we can use the
+    # globally installed Python toolchain.
+    #export PATH=/opt/python/3.7/bin:$PATH
+
+    # 12.04, 14.04 and 16.04 are good
+    curl --retry 3 -fL http://rubies.travis-ci.org/ubuntu/`lsb_release -rs`/x86_64/ruby-head.tar.bz2 |tar xfj -
+    # TODO adjust gem path?
+    export PATH=`pwd`/ruby-head/bin:`pwd`/ruby-head/lib/ruby/gems/2.6.0/bin:$PATH
+    ruby --version
+    ruby --version |grep dev
+  else
+    if test "$USE_OPT_TOOLCHAIN" = 1; then
+      # nothing, also PATH is already set
+      :
+    elif true; then
+
+    # For testing toolchains:
+    #toolchain_url=https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/`host_distro`/f11598d091441ffc8d746aacfdc6c26741a3e629/mongo_ruby_driver_toolchain_`host_distro |tr - _`_patch_f11598d091441ffc8d746aacfdc6c26741a3e629_5e46f2793e8e866f36eda2c5_20_02_14_19_18_18.tar.gz
+    toolchain_url=http://boxes.10gen.com/build/toolchain-drivers/mongo-ruby-driver/ruby-toolchain-`host_distro`-$TOOLCHAIN_VERSION.tar.xz
+    curl --retry 3 -fL $toolchain_url |tar Jxf -
+    export PATH=`pwd`/rubies/$RVM_RUBY/bin:$PATH
+    #export PATH=`pwd`/rubies/python/3/bin:$PATH
+
+    # Attempt to get bundler to report all errors - so far unsuccessful
+    #curl --retry 3 -o bundler-openssl.diff https://github.com/bundler/bundler/compare/v2.0.1...p-mongo:report-errors.diff
+    #find . -path \*/lib/bundler/fetcher.rb -exec patch {} bundler-openssl.diff \;
+
+    else
+
+    # Normal operation
+    if ! test -d $HOME/.rubies/$RVM_RUBY/bin; then
+      echo "Ruby directory does not exist: $HOME/.rubies/$RVM_RUBY/bin" 1>&2
+      echo "Contents of /opt:" 1>&2
+      ls -l /opt 1>&2 || true
+      echo ".rubies symlink:" 1>&2
+      ls -ld $HOME/.rubies 1>&2 || true
+      echo "Our rubies:" 1>&2
+      ls -l $HOME/.rubies 1>&2 || true
+      exit 2
+    fi
+    export PATH=$HOME/.rubies/$RVM_RUBY/bin:$PATH
+
+    fi
+
+    ruby --version
+
+    # Ensure we're using the right ruby
+    ruby_name=`echo $RVM_RUBY |awk -F- '{print $1}'`
+    ruby_version=`echo $RVM_RUBY |awk -F- '{print $2}' |cut -c 1-3`
+
+    ruby -v |fgrep $ruby_name
+    ruby -v |fgrep $ruby_version
+
+    # We shouldn't need to update rubygems, and there is value in
+    # testing on whatever rubygems came with each supported ruby version
+    #echo 'updating rubygems'
+    #gem update --system
+
+    # Only install bundler when not using ruby-head.
+    # ruby-head comes with bundler and gem complains
+    # because installing bundler would overwrite the bundler binary.
+    # We now install bundler in the toolchain, hence nothing needs to be done
+    # in the tests.
+    if false && echo "$RVM_RUBY" |grep -q jruby; then
+      gem install bundler -v '<2'
+    fi
+  fi
+}

data/spec/spec_helper.rb

@@ -41,7 +41,15 @@ unless ENV['CI'] || BSON::Environment.jruby?
   end
 end
 
-require 'mrss/lite_constraints'
+begin
+  require 'mrss/lite_constraints'
+rescue LoadError => exc
+  raise LoadError.new <<~MSG.strip
+    The test suite requires shared tooling to be installed.
+      Please refer to spec/README.md for instructions.
+    #{exc.class}: #{exc}
+  MSG
+end
 
 Dir["./spec/support/**/*.rb"].each { |file| require file }
 

data/spec/spec_tests/data/corpus/binary.json

@@ -50,6 +50,11 @@
             "canonical_bson": "1D000000057800100000000573FFD26444B34C6990E8E7D1DFC035D400",
             "canonical_extjson": "{\"x\" : { \"$binary\" : {\"base64\" : \"c//SZESzTGmQ6OfR38A11A==\", \"subType\" : \"05\"}}}"
         },
+        {
+            "description": "subtype 0x07",
+            "canonical_bson": "1D000000057800100000000773FFD26444B34C6990E8E7D1DFC035D400",
+            "canonical_extjson": "{\"x\" : { \"$binary\" : {\"base64\" : \"c//SZESzTGmQ6OfR38A11A==\", \"subType\" : \"07\"}}}"
+        },
         {
             "description": "subtype 0x80",
             "canonical_bson": "0F0000000578000200000080FFFF00",

data/spec/spec_tests/data/corpus/dbref.json

@@ -1,5 +1,5 @@
 {
-    "description": "DBRef",
+    "description": "Document type (DBRef sub-documents)",
     "bson_type": "0x03",
     "valid": [
         {
@@ -26,6 +26,26 @@
             "description": "Document with key names similar to those of a DBRef",
             "canonical_bson": "3e0000000224726566000c0000006e6f742d612d646272656600072469640058921b3e6e32ab156a22b59e022462616e616e6100050000007065656c0000",
             "canonical_extjson": "{\"$ref\": \"not-a-dbref\", \"$id\": {\"$oid\": \"58921b3e6e32ab156a22b59e\"}, \"$banana\": \"peel\"}"
+        },
+        {
+            "description": "DBRef with additional dollar-prefixed and dotted fields",
+            "canonical_bson": "48000000036462726566003c0000000224726566000b000000636f6c6c656374696f6e00072469640058921b3e6e32ab156a22b59e10612e62000100000010246300010000000000",
+            "canonical_extjson": "{\"dbref\": {\"$ref\": \"collection\", \"$id\": {\"$oid\": \"58921b3e6e32ab156a22b59e\"}, \"a.b\": {\"$numberInt\": \"1\"}, \"$c\": {\"$numberInt\": \"1\"}}}"
+        },
+        {
+            "description": "Sub-document resembles DBRef but $id is missing",
+            "canonical_bson": "26000000036462726566001a0000000224726566000b000000636f6c6c656374696f6e000000",
+            "canonical_extjson": "{\"dbref\": {\"$ref\": \"collection\"}}"
+        },
+        {
+            "description": "Sub-document resembles DBRef but $ref is not a string",
+            "canonical_bson": "2c000000036462726566002000000010247265660001000000072469640058921b3e6e32ab156a22b59e0000",
+            "canonical_extjson": "{\"dbref\": {\"$ref\": {\"$numberInt\": \"1\"}, \"$id\": {\"$oid\": \"58921b3e6e32ab156a22b59e\"}}}"
+        },
+        {
+            "description": "Sub-document resembles DBRef but $db is not a string",
+            "canonical_bson": "4000000003646272656600340000000224726566000b000000636f6c6c656374696f6e00072469640058921b3e6e32ab156a22b59e1024646200010000000000",
+            "canonical_extjson": "{\"dbref\": {\"$ref\": \"collection\", \"$id\": {\"$oid\": \"58921b3e6e32ab156a22b59e\"}, \"$db\": {\"$numberInt\": \"1\"}}}"
         }
     ]
 }

data/spec/spec_tests/data/corpus/document.json

@@ -31,6 +31,10 @@
         {
             "description": "Invalid subdocument: bad string length in field",
             "bson": "1C00000003666F6F001200000002626172000500000062617A000000"
+        },
+        {
+            "description": "Null byte in sub-document key",
+            "bson": "150000000378000D00000010610000010000000000"
         }
     ]
 }

data/spec/spec_tests/data/corpus/regex.json

@@ -54,11 +54,11 @@
     ],
     "decodeErrors": [
         {
-            "description": "embedded null in pattern",
+            "description": "Null byte in pattern string",
             "bson": "0F0000000B610061006300696D0000"
         },
         {
-            "description": "embedded null in flags",
+            "description": "Null byte in flags string",
             "bson": "100000000B61006162630069006D0000"
         }
     ]

data/spec/spec_tests/data/corpus/top.json

@@ -60,6 +60,10 @@
         {
             "description": "Document truncated mid-key",
             "bson": "1200000002666F"
+        },
+        {
+            "description": "Null byte in document key",
+            "bson": "0D000000107800000100000000"
         }
     ],
     "parseErrors": [
@@ -195,14 +199,6 @@
             "description": "Bad $date (extra field)",
             "string": "{\"a\" : {\"$date\" : {\"$numberLong\" : \"1356351330501\"}, \"unrelated\": true}}"
         },
-        {
-            "description": "Bad DBRef (ref is number, not string)",
-            "string": "{\"x\" : {\"$ref\" : 42, \"$id\" : \"abc\"}}"
-        },
-        {
-            "description": "Bad DBRef (db is number, not string)",
-            "string": "{\"x\" : {\"$ref\" : \"a\", \"$id\" : \"abc\", \"$db\" : 42}}"
-        },
         {
             "description": "Bad $minKey (boolean, not integer)",
             "string": "{\"a\" : {\"$minKey\" : true}}"
@@ -230,7 +226,22 @@
         {
             "description": "Bad DBpointer (extra field)",
             "string": "{\"a\": {\"$dbPointer\": {\"a\": {\"$numberInt\": \"1\"}, \"$id\": {\"$oid\": \"56e1fc72e0c917e9c4714161\"}, \"c\": {\"$numberInt\": \"2\"}, \"$ref\": \"b\"}}}"
+        },
+        {
+            "description" : "Null byte in document key",
+            "string" : "{\"a\\u0000\": 1 }"
+        },
+        {
+            "description" : "Null byte in sub-document key",
+            "string" : "{\"a\" : {\"b\\u0000\": 1 }}"
+        },
+        {
+            "description": "Null byte in $regularExpression pattern",
+            "string": "{\"a\" : {\"$regularExpression\" : { \"pattern\": \"b\\u0000\", \"options\" : \"i\"}}}"
+        },
+        {
+            "description": "Null byte in $regularExpression options",
+            "string": "{\"a\" : {\"$regularExpression\" : { \"pattern\": \"b\", \"options\" : \"i\\u0000\"}}}"
         }
-
     ]
 }