bson 4.11.0 → 4.13.0

data/spec/shared/lib/mrss/lite_constraints.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+module Mrss
+  module LiteConstraints
+
+    # Constrain tests that use TimeoutInterrupt to MRI (and Unix).
+    def require_mri
+      before(:all) do
+        unless SpecConfig.instance.mri?
+          skip "MRI required, we have #{SpecConfig.instance.platform}"
+        end
+      end
+    end
+
+    def require_jruby
+      before(:all) do
+        unless BSON::Environment.jruby?
+          skip "JRuby required, we have #{SpecConfig.instance.platform}"
+        end
+      end
+    end
+
+    # This is for marking tests that fail on JRuby that should
+    # in principle work (as opposed to being fundamentally incompatible
+    # with JRuby).
+    # Often times these failures happen only in Evergreen.
+    def fails_on_jruby(version=nil)
+      before(:all) do
+        if BSON::Environment.jruby?
+          if version
+            min_parts = version.split('.').map(&:to_i)
+            actual_parts = JRUBY_VERSION.split('.').map(&:to_i)[0...min_parts.length]
+            actual = actual_parts.join('.')
+            if actual <= version
+              skip "Fails on jruby through #{version}"
+            end
+          else
+            skip "Fails on jruby"
+          end
+        end
+      end
+    end
+
+    # Indicates that the respective test uses the internet in some capacity,
+    # for example the test resolves SRV DNS records.
+    def require_external_connectivity
+      before(:all) do
+        if ENV['EXTERNAL_DISABLED']
+          skip "Test requires external connectivity"
+        end
+      end
+    end
+
+    def require_mongo_kerberos
+      before(:all) do
+        # TODO Use a more generic environment variable name if/when
+        # Mongoid tests get Kerberos configurations.
+        unless %w(1 yes true).include?(ENV['MONGO_RUBY_DRIVER_KERBEROS']&.downcase)
+          skip 'Set MONGO_RUBY_DRIVER_KERBEROS=1 in environment to run Kerberos unit tests'
+        end
+        require 'mongo_kerberos'
+      end
+    end
+
+    def require_linting
+      before(:all) do
+        unless Mongo::Lint.enabled?
+          skip "Linting is not enabled"
+        end
+      end
+    end
+
+    # Some tests will fail if linting is enabled:
+    # 1. Tests that pass invalid options to client, etc. which the linter
+    #    rejects.
+    # 2. Tests that set expectations on topologies, server descriptions, etc.
+    #    (since setting expectations requires mutating said objects, and when
+    #    linting is on those objects are frozen).
+    def require_no_linting
+      before(:all) do
+        if Mongo::Lint.enabled?
+          skip "Linting is enabled"
+        end
+      end
+    end
+
+    def require_libmongocrypt
+      before(:all) do
+        unless ENV['LIBMONGOCRYPT_PATH']
+          skip 'Test requires path to libmongocrypt to be specified in LIBMONGOCRYPT_PATH env variable'
+        end
+      end
+    end
+
+    def require_no_libmongocrypt
+      before(:all) do
+        if ENV['LIBMONGOCRYPT_PATH']
+          skip 'Test requires libmongocrypt to not be configured'
+        end
+      end
+    end
+
+    def require_aws_auth
+      before(:all) do
+        unless (ENV['AUTH'] || '') =~ /^aws/
+          skip 'This test requires AUTH=aws* and an appropriately configured runtime environment'
+        end
+      end
+    end
+
+    def require_ec2_host
+      before(:all) do
+        if $have_aws.nil?
+          $have_aws = begin
+            require 'open-uri'
+            begin
+              Timeout.timeout(3.81) do
+                URI.parse('http://169.254.169.254/latest/meta-data/profile').open.read
+              end
+              true
+            # When trying to use the EC2 metadata endpoint on ECS:
+            # Errno::EINVAL: Failed to open TCP connection to 169.254.169.254:80 (Invalid argument - connect(2) for "169.254.169.254" port 80)
+            rescue Timeout::Error, Errno::ETIMEDOUT, Errno::EINVAL, OpenURI::HTTPError => $aws_error
+              false
+            end
+          end
+        end
+        unless $have_aws
+          skip "EC2 instance metadata is not available - assuming not running on an EC2 instance: #{$aws_error.class}: #{$aws_error}"
+        end
+      end
+    end
+
+    def require_stress
+      before(:all) do
+        if !SpecConfig.instance.stress?
+          skip 'Set STRESS=1 in environment to run stress tests'
+        end
+      end
+    end
+
+    def require_fork
+      before(:all) do
+        if !SpecConfig.instance.fork?
+          skip 'Set FORK=1 in environment to run fork tests'
+        end
+      end
+    end
+
+    def require_ocsp
+      before(:all) do
+        if !SpecConfig.instance.ocsp?
+          skip 'Set OCSP=1 in environment to run OCSP tests'
+        end
+      end
+    end
+
+    def require_ocsp_verifier
+      before(:all) do
+        if !SpecConfig.instance.ocsp_verifier?
+          skip 'Set OCSP_VERIFIER=1 in environment to run OCSP verifier tests'
+        end
+      end
+    end
+
+    def require_ocsp_connectivity
+      before(:all) do
+        if !SpecConfig.instance.ocsp_connectivity?
+          skip 'Set OCSP_CONNECTIVITY=pass or OCSP_CONNECTIVITY=fail in environment to run OCSP connectivity tests'
+        end
+      end
+    end
+
+    def require_active_support
+      before(:all) do
+        if !SpecConfig.instance.active_support?
+          skip 'This test requires ActiveSupport; set WITH_ACTIVE_SUPPORT=1 in environment'
+        end
+      end
+    end
+
+    def no_active_support
+      before(:all) do
+        if SpecConfig.instance.active_support?
+          skip 'This test requires no ActiveSupport; unset WITH_ACTIVE_SUPPORT in environment'
+        end
+      end
+    end
+  end
+end

data/spec/shared/lib/mrss/server_version_registry.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+autoload :JSON, 'json'
+require 'open-uri'
+
+module Mrss
+  class ServerVersionRegistry
+    class Error < StandardError
+    end
+
+    class UnknownVersion < Error
+    end
+
+    class MissingDownloadUrl < Error
+    end
+
+    class BrokenDownloadUrl < Error
+    end
+
+    def initialize(desired_version, arch)
+      @desired_version, @arch = desired_version, arch
+    end
+
+    attr_reader :desired_version, :arch
+
+    def download_url
+      @download_url ||= begin
+        version, version_ok = detect_version(current_catalog)
+        if version.nil?
+          version, full_version_ok = detect_version(full_catalog)
+          version_ok ||= full_version_ok
+        end
+        if version.nil?
+          if version_ok
+            raise MissingDownloadUrl, "No downloads for version #{desired_version}"
+          else
+            raise UnknownVersion, "No version #{desired_version}"
+          end
+        end
+        dl = version['downloads'].detect do |dl|
+          dl['archive']['url'].index("enterprise-#{arch}") &&
+          dl['arch'] == 'x86_64'
+        end
+        unless dl
+          raise MissingDownloadUrl, "No download for #{arch} for #{version['version']}"
+        end
+        url = dl['archive']['url']
+      end
+    rescue MissingDownloadUrl
+      if %w(4.7 4.7.0).include?(desired_version)
+        # 4.7.0 has no advertised downloads but it is downloadable and
+        # we do need it. Dirty hack below.
+        registry = self.class.new('4.4.3', arch)
+        registry.download_url.sub('4.4.3', '4.7.0').tap do |url|
+          # Sanity check - ensure the URL we hacked up is a valid one
+          io = uri_open(url)
+          begin
+            io.read(1)
+          ensure
+            io.close
+          end
+        end
+      else
+        raise
+      end
+    end
+
+    private
+
+    def uri_open(*args)
+      if RUBY_VERSION < '2.5'
+        open(*args)
+      else
+        URI.open(*args)
+      end
+    end
+
+    def detect_version(catalog)
+      candidate_versions = catalog['versions'].select do |version|
+        version['version'].start_with?(desired_version) &&
+        !version['version'].include?('-')
+      end
+      version_ok = !candidate_versions.empty?
+      # Sometimes the download situation is borked and there is a release
+      # with no downloads... skip those.
+      version = candidate_versions.detect do |version|
+        !version['downloads'].empty?
+      end
+      # Allow RC releases if there isn't a GA release.
+      if version.nil?
+        candidate_versions = catalog['versions'].select do |version|
+          version['version'].start_with?(desired_version)
+        end
+        version_ok ||= !candidate_versions.empty?
+        version = candidate_versions.detect do |version|
+          !version['downloads'].empty?
+        end
+      end
+      [version, version_ok]
+    end
+
+    def current_catalog
+      @current_catalog ||= begin
+        JSON.load(uri_open('http://downloads.mongodb.org/current.json').read)
+      end
+    end
+
+    def full_catalog
+      @full_catalog ||= begin
+        JSON.load(uri_open('http://downloads.mongodb.org/full.json').read)
+      end
+    end
+  end
+end

data/spec/shared/lib/mrss/spec_organizer.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+autoload :JSON, 'json'
+autoload :FileUtils, 'fileutils'
+autoload :Find, 'find'
+
+module Mrss
+
+  autoload :ChildProcessHelper, 'mrss/child_process_helper'
+
+  # Organizes and runs all of the tests in the test suite in batches.
+  #
+  # Organizing the tests in batches serves two purposes:
+  #
+  # 1. This allows running unit tests before integration tests, therefore
+  #    in theory revealing failures quicker on average.
+  # 2. This allows running some tests that have high intermittent failure rate
+  #    in their own test process.
+  #
+  # This class aggregates RSpec results after the test runs.
+  class SpecOrganizer
+
+    class BucketsNotPrioritized < StandardError
+    end
+
+    def initialize(root: nil, classifiers:, priority_order:,
+      spec_root: nil, rspec_json_path: nil, rspec_all_json_path: nil
+    )
+      @spec_root = spec_root || File.join(root, 'spec')
+      @classifiers = classifiers
+      @priority_order = priority_order
+      @rspec_json_path = rspec_json_path || File.join(root, 'tmp/rspec.json')
+      @rspec_all_json_path = rspec_all_json_path || File.join(root, 'tmp/rspec-all.json')
+    end
+
+    attr_reader :spec_root, :classifiers, :priority_order
+    attr_reader :rspec_json_path, :rspec_all_json_path
+
+    def buckets
+      @buckets ||= {}.tap do |buckets|
+        Find.find(spec_root) do |path|
+          next unless File.file?(path)
+          next unless path =~ /_spec\.rb\z/
+          rel_path = path[(spec_root.length + 1)..path.length]
+
+          found = false
+          classifiers.each do |(regexp, category)|
+            if regexp =~ rel_path
+              buckets[category] ||= []
+              buckets[category] << File.join('spec', rel_path)
+              found = true
+              break
+            end
+          end
+
+          unless found
+            buckets[nil] ||= []
+            buckets[nil] << File.join('spec', rel_path)
+          end
+        end
+      end.freeze
+    end
+
+    def ordered_buckets
+      @ordered_buckets ||= {}.tap do |ordered_buckets|
+        buckets = self.buckets.dup
+        priority_order.each do |category|
+          files = buckets.delete(category)
+          ordered_buckets[category] = files
+        end
+
+        if files = buckets.delete(nil)
+          ordered_buckets[nil] = files
+        end
+
+        unless buckets.empty?
+          raise BucketsNotPrioritized, "Some buckets were not prioritized: #{buckets.keys.map(&:to_s).join(', ')}"
+        end
+      end.freeze
+    end
+
+    def run
+      FileUtils.rm_f(rspec_all_json_path)
+
+      failed = []
+      buckets = self.buckets.dup
+
+      priority_order.each do |category|
+        if files = buckets.delete(category)
+          unless run_files(category, files)
+            failed << category
+          end
+        end
+      end
+      if files = buckets.delete(nil)
+        unless run_files('remaining', files)
+          failed << 'remaining'
+        end
+      end
+
+      unless buckets.empty?
+        raise "Some buckets were not executed: #{buckets.keys.map(&:to_s).join(', ')}"
+      end
+
+      if failed.any?
+        raise "The following buckets failed: #{failed.map(&:to_s).join(', ')}"
+      end
+    end
+
+    def run_files(category, paths)
+      puts "Running #{category.to_s.gsub('_', ' ')} tests"
+      FileUtils.rm_f(rspec_json_path)
+      cmd = %w(rspec) + paths
+
+      begin
+        ChildProcessHelper.check_call(cmd)
+      ensure
+        if File.exist?(rspec_json_path)
+          if File.exist?(rspec_all_json_path)
+            merge_rspec_results
+          else
+            FileUtils.cp(rspec_json_path, rspec_all_json_path)
+          end
+        end
+      end
+
+      true
+    rescue ChildProcessHelper::SpawnError
+      false
+    end
+
+    def merge_rspec_results
+      all = JSON.parse(File.read(rspec_all_json_path))
+      new = JSON.parse(File.read(rspec_json_path))
+      all['examples'] += new.delete('examples')
+      new.delete('summary').each do |k, v|
+        all['summary'][k] += v
+      end
+      new.delete('version')
+      new.delete('summary_line')
+      unless new.empty?
+        raise "Unhandled rspec results keys: #{new.keys.join(', ')}"
+      end
+      # We do not merge summary lines, delete them from aggregated results
+      all.delete('summary_line')
+      File.open(rspec_all_json_path, 'w') do |f|
+        f << JSON.dump(all)
+      end
+    end
+  end
+end

data/spec/shared/lib/mrss/utils.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+module Mrss
+  module Utils
+
+    module_function def print_backtrace(dest=STDERR)
+      begin
+        hello world
+      rescue => e
+        dest.puts e.backtrace.join("\n")
+      end
+    end
+  end
+end

data/spec/shared/share/Dockerfile.erb

@@ -0,0 +1,231 @@
+# Python toolchain as of this writing is available on rhel62, debian92 and
+# ubuntu1604.
+#
+# To run rhel62 in docker, host system must be configured to emulate syscalls:
+# https://github.com/CentOS/sig-cloud-instance-images/issues/103
+
+<%
+
+python_toolchain_url = "https://s3.amazonaws.com//mciuploads/mongo-python-driver-toolchain/#{distro}/ba92de2700c04ee2d4f82c3ffdfc33105140cb04/mongo_python_driver_toolchain_#{distro.gsub('-', '_')}_ba92de2700c04ee2d4f82c3ffdfc33105140cb04_19_11_14_15_33_33.tar.gz"
+server_version = '4.3.3'
+server_url = "http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-#{distro}-#{server_version}.tgz"
+server_archive_basename = File.basename(server_url)
+server_extracted_dir = server_archive_basename.sub(/\.(tar\.gz|tgz)$/, '')
+
+toolchain_upper='291ba4a4e8297f142796e70eee71b99f333e35e1'
+
+ruby_toolchain_url = "http://boxes.10gen.com/build/toolchain-drivers/mongo-ruby-driver/ruby-toolchain-#{distro}-#{toolchain_upper}.tar.xz"
+#ruby_toolchain_url = "https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/#{distro}/#{toolchain_upper}/mongo_ruby_driver_toolchain_#{distro.gsub('-', '_')}_patch_#{toolchain_upper}_#{toolchain_lower}.tar.gz"
+
+%>
+
+FROM <%= base_image %>
+
+<% if debian? %>
+
+  ENV DEBIAN_FRONTEND=noninteractive
+
+<% else %>
+
+  RUN echo assumeyes=1 |tee -a /etc/yum.conf
+
+<% end %>
+
+<% if ruby_head? %>
+
+  # To use current versions of mlaunch, Python 3.6+ is required.
+  # Most distros ship with older Pythons, therefore we need to install
+  # a newer Python from somewhere. This section installs the Python
+  # toolhcain which comes with recent Pythons.
+  # Alternatively, Ruby toolchain compiles its own copy of Python 3 but
+  # this is currently incomplete in that on older distros with old OpenSSL,
+  # the built Python has no ssl module and hence practically is unusable.
+  # Currently Ruby driver uses mtools-legacy which supports Python 2,
+  # avoiding this entire issue for the time being.
+
+  #RUN curl --retry 3 -fL <%= python_toolchain_url %> -o python-toolchain.tar.gz
+  #RUN tar -xC /opt -zf python-toolchain.tar.gz
+
+<% end %>
+
+<% if debian? %>
+
+  # zsh is not required for any scripts but it is a better interactive shell
+  # than bash.
+  # Ruby runtime dependencies: libyaml-0-2
+  # Compiling ruby libraries: gcc make
+  # Compiling pyhton packages: python2.7-dev
+  # JRuby: openjdk-8-jre-headless
+  # Server dependencies: libsnmp30 libcurl3/libcurl4
+  # Determining OS we are running on: lsb-release
+  # Kerberos testing: krb5-user
+  # Local Kerberos server: krb5-kdc krb5-admin-server
+  # Installing mlaunch from git: git
+  # ruby-head archive: bzip2
+  # nio4r on JRuby: libgmp-dev
+  # Snappy compression: libsnappy-dev
+  # nokogiri: zlib1g-dev
+  # Mongoid testing: tzdata
+  # Mongoid application testing: nodejs (8.x or newer)
+  #
+  # We currently use Python 2-compatible version of mtools, which
+  # is installable via pip (which uses Python 2). All of the MongoDB
+  # distros have pip installed (but none as of this writing have pip3)
+  # therefore install python-pip in all configurations here.
+
+  <% packages = %w(
+    lsb-release bzip2 curl zsh
+    git make gcc libyaml-0-2 libgmp-dev zlib1g-dev libsnappy-dev
+    libsnmp30
+    krb5-user krb5-kdc krb5-admin-server libsasl2-dev libsasl2-modules-gssapi-mit
+    python-pip python2.7-dev python3-pip
+    tzdata
+  ) %>
+
+  # ubuntu1404 only has openjdk-7-jre-headless
+  <% if distro !~ /ubuntu1404/ %>
+    <% packages << 'openjdk-8-jre-headless' %>
+  <% end %>
+
+  # ubuntu1404, ubuntu1604: libcurl3
+  # ubuntu1804: libcurl4
+  <% if distro =~ /ubuntu1804/ %>
+    <% packages << 'libcurl4' %>
+  <% else %>
+    <% packages << 'libcurl3' %>
+  <% end %>
+
+  <% if distro =~ /ubuntu1804/ %>
+    <% packages << 'nodejs' %>
+  <% end %>
+
+  RUN apt-get update && apt-get install -y <%= packages.join(' ') %>
+<% else %>
+
+  # Enterprise server: net-snmp
+  # lsb_release: redhat-lsb-core
+  # our runner scripts: which
+  # Ruby dependency: libyaml
+  # compiling python packages: gcc python-devel
+  # Kerberos tests: krb5-workstation + cyrus-sasl-devel to build the
+  # mongo_kerberos gem + cyrus-sasl-gssapi for authentication to work
+  # Local Kerberos server: krb5-server
+  # JRuby: java-1.8.0-openjdk
+  #
+  # Note: lacking cyrus-sasl-gssapi produces a cryptic message
+  # "SASL(-4): no mechanism available: No worthy mechs found"
+  # https://github.com/farorm/python-ad/issues/10
+
+  RUN yum install -y redhat-lsb-core which git gcc libyaml krb5-server \
+    krb5-workstation cyrus-sasl-devel cyrus-sasl-gssapi java-1.8.0-openjdk \
+    net-snmp
+
+  <% if distro =~ /rhel6/ %>
+
+    # RHEL 6 ships with Python 2.6.
+
+    RUN yum install -y centos-release-scl && \
+      yum install -y python27-python python27-python-devel
+    ENV PATH=/opt/rh/python27/root/usr/bin:$PATH \
+      LD_LIBRARY_PATH=/opt/rh/python27/root/usr/lib64
+
+  <% else %>
+
+    RUN yum install -y python-devel
+
+  <% end %>
+
+<% end %>
+
+<% if preload? %>
+
+  # Current virtualenv fails with
+  # https://github.com/pypa/virtualenv/issues/1630
+  RUN python2 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]'
+
+  RUN pip --version && \
+    pip install mtools-legacy[mlaunch]
+
+  <% if @env.fetch('MONGODB_VERSION') >= '4.4' %>
+    RUN python3 -mpip install asn1crypto oscrypto flask
+  <% end %>
+
+  <% unless ruby_head? %>
+
+    RUN curl --retry 3 -fL <%= ruby_toolchain_url %> |tar -xC /opt -Jf -
+    ENV PATH=/opt/rubies/<%= ruby %>/bin:$PATH \
+      USE_OPT_TOOLCHAIN=1
+    #ENV PATH=/opt/rubies/python/3/bin:$PATH
+
+  <% end %>
+
+  <% if distro =~ /rhel|ubuntu1604/ %>
+
+    # Ubuntu 12.04 ships pip 1.0 which is ancient and does not work.
+    #
+    # Ubuntu 16.04 apparently also ships a pip that does not work:
+    # https://stackoverflow.com/questions/37495375/python-pip-install-throws-typeerror-unsupported-operand-types-for-retry
+    # Potentially this only affects environments with less than ideal
+    # connectivity (or, perhaps, when python package registry is experiencing
+    # availability issues) when pip must retry to install packages.
+    #
+    # rhel apparently does not package pip at all in core repoitories,
+    # therefore install it the manual way.
+    #
+    # https://pip.pypa.io/en/stable/installing/
+    RUN curl --retry 3 -fL https://bootstrap.pypa.io/pip/2.7/get-pip.py | python
+
+  <% end %>
+
+  RUN curl --retry 3 -fL <%= server_download_url %> |tar xzf - && \
+    mv mongo*/ /opt/mongodb
+  ENV USE_OPT_MONGODB=1 USE_SYSTEM_PYTHON_PACKAGES=1
+
+<% end %>
+
+WORKDIR /app
+
+<% if preload? && !ruby_head? %>
+
+  COPY Gemfile .
+  COPY gemfiles gemfiles
+  COPY *.gemspec .
+  COPY lib/<%= project_lib_subdir %>/version.rb lib/<%= project_lib_subdir %>/version.rb
+  RUN bundle install
+  COPY .evergreen/patch-debuggers .evergreen/patch-debuggers
+  RUN .evergreen/patch-debuggers /opt/rubies
+
+<% end %>
+
+<% if fle? %>
+  RUN curl --retry 3 -fLo libmongocrypt-all.tar.gz "https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz"
+  RUN tar xf libmongocrypt-all.tar.gz
+
+  <%= "ENV LIBMONGOCRYPT_PATH #{libmongocrypt_path}" %>
+<% end %>
+
+ENV MONGO_ORCHESTRATION_HOME=/tmpfs \
+  PROJECT_DIRECTORY=/app \
+  <%= @env.map { |k, v| %Q`#{k}="#{v.gsub('$', "\\$").gsub('"', "\\\"")}"` }.join(" \\\n  ") %>
+
+<% if interactive? %>
+  ENV INTERACTIVE=1
+<% end %>
+
+COPY . .
+
+<% if expose? %>
+
+  <% ports = [] %>
+
+  <% 0.upto(num_exposed_ports-1) do |i| %>
+    <% ports << 27017 + i %>
+  <% end %>
+
+  <% if @env['OCSP_ALGORITHM'] %>
+    <% ports << 8100 %>
+  <% end %>
+
+  EXPOSE <%= ports.map(&:to_s).join(' ') %>
+
+<% end %>