mongo 2.16.3 → 2.17.0

data/spec/runners/unified/ddl_operations.rb

@@ -7,7 +7,13 @@ module Unified
 
     def list_databases(op)
       client = entities.get(:client, op.use!('object'))
-      client.list_databases
+      use_arguments(op) do |args|
+        opts = {}
+        if session = args.use('session')
+          opts[:session] = entities.get(:session, session)
+        end
+        client.list_databases({}, false, **opts)
+      end
     end
 
     def create_collection(op)
@@ -28,6 +34,17 @@ module Unified
       end
     end
 
+    def list_collections(op)
+      database = entities.get(:database, op.use!('object'))
+      use_arguments(op) do |args|
+        opts = {}
+        if session = args.use('session')
+          opts[:session] = entities.get(:session, session)
+        end
+        database.list_collections(**opts)
+      end
+    end
+
     def drop_collection(op)
       database = entities.get(:database, op.use!('object'))
       use_arguments(op) do |args|
@@ -58,6 +75,17 @@ module Unified
       assert_collection_exists(op, false)
     end
 
+    def list_indexes(op)
+      collection = entities.get(:collection, op.use!('object'))
+      use_arguments(op) do |args|
+        opts = {}
+        if session = args.use('session')
+          opts[:session] = entities.get(:session, session)
+        end
+        collection.indexes(**opts).to_a
+      end
+    end
+
     def create_index(op)
       collection = entities.get(:collection, op.use!('object'))
       use_arguments(op) do |args|

data/spec/runners/unified/entity_map.rb

@@ -3,6 +3,8 @@
 
 module Unified
   class EntityMap
+    extend Forwardable
+
     def initialize
       @map = {}
     end
@@ -35,8 +37,6 @@ module Unified
       raise Error::EntityMissing, "There is no #{id} known"
     end
 
-    def [](type)
-      @map[type]
-    end
+    def_delegators :@map, :[], :fetch
   end
 end

data/spec/runners/unified/support_operations.rb

@@ -13,7 +13,12 @@ module Unified
 
         cmd = args.use!('command')
 
-        database.command(cmd)
+        opts = {}
+        if session = args.use('session')
+          opts[:session] = entities.get(:session, session)
+        end
+
+        database.command(cmd, **opts)
       end
     end
 

data/spec/runners/unified/test.rb

@@ -160,13 +160,24 @@ module Unified
           end
         when 'database'
           client = entities.get(:client, spec.use!('client'))
-          client.use(spec.use!('databaseName')).database
+          opts = Utils.snakeize_hash(spec.use('databaseOptions') || {})
+            .merge(database: spec.use!('databaseName'))
+          if opts.key?(:read_preference)
+            opts[:read] = opts.delete(:read_preference)
+            if opts[:read].key?(:max_staleness_seconds)
+              opts[:read][:max_staleness] = opts[:read].delete(:max_staleness_seconds)
+            end
+          end
+          client.with(opts).database
         when 'collection'
           database = entities.get(:database, spec.use!('database'))
           # TODO verify
           opts = Utils.snakeize_hash(spec.use('collectionOptions') || {})
           if opts.key?(:read_preference)
             opts[:read] = opts.delete(:read_preference)
+            if opts[:read].key?(:max_staleness_seconds)
+              opts[:read][:max_staleness] = opts[:read].delete(:max_staleness_seconds)
+            end
           end
           database[spec.use!('collectionName'), opts]
         when 'bucket'
@@ -195,7 +206,8 @@ module Unified
     def set_initial_data
       @spec['initialData']&.each do |entity_spec|
         spec = UsingHash[entity_spec]
-        collection = root_authorized_client.use(spec.use!('databaseName'))[spec.use!('collectionName')]
+        collection = root_authorized_client.with(write_concern: {w: :majority}).
+          use(spec.use!('databaseName'))[spec.use!('collectionName')]
         collection.drop
         docs = spec.use!('documents')
         if docs.any?
@@ -268,7 +280,7 @@ module Unified
         end
         if expected_error = op.use('expectError')
           begin
-            send(method_name, op)
+            public_send(method_name, op)
           rescue Mongo::Error, BSON::String::IllegalKey => e
             if expected_error.use('isClientError')
               # isClientError doesn't actually mean a client error.

data/spec/runners/unified/test_group.rb

@@ -6,7 +6,7 @@ module Unified
   class TestGroup
     def initialize(path, **opts)
       if String === path
-        data = ::Utils.load_spec_yaml_file(path)
+        data = YAML.load(File.read(path))
       else
         data = path
       end

data/spec/shared/share/Dockerfile.erb

@@ -233,13 +233,13 @@ CFG
   # Current virtualenv fails with
   # https://github.com/pypa/virtualenv/issues/1630
   <% if distro =~ /ubuntu2004/ %>
-    RUN python3 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]==1.5.5'
+    RUN python3 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]'
   <% else %>
-    RUN python2 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]==1.5.5'
+    RUN python2 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]'
   <% end %>
 
   RUN pip --version && \
-    pip install 'mtools-legacy[mlaunch]==1.5.5'
+    pip install mtools-legacy[mlaunch]
 
   <% if @env.fetch('MONGODB_VERSION') >= '4.4' %>
     # ubuntu1604 installs MarkupSafe 0.0.0 here instead of 2.0.0+

data/spec/shared/shlib/server.sh

@@ -71,7 +71,7 @@ install_mlaunch_virtualenv() {
     venvpath="$MONGO_ORCHESTRATION_HOME"/venv
     python2 -m virtualenv -p python2 $venvpath
     . $venvpath/bin/activate
-    pip install 'mtools-legacy[mlaunch]==1.5.5'
+    pip install 'mtools-legacy[mlaunch]'
   fi
 }
 

data/spec/spec_tests/data/crud_unified/aggregate-let.yml

@@ -0,0 +1,138 @@
+description: "aggregate-let"
+
+schemaVersion: "1.4"
+
+createEntities:
+  - client:
+      id: &client0 client0
+      observeEvents: [ commandStartedEvent ]
+  - database:
+      id: &database0 database0
+      client: *client0
+      databaseName: &database0Name crud-tests
+  - collection:
+      id: &collection0 collection0
+      database: *database0
+      collectionName: &collection0Name coll0
+  - collection:
+      id: &collection1 collection1
+      database: *database0
+      collectionName: &collection1Name coll1
+
+initialData: &initialData
+  - collectionName: *collection0Name
+    databaseName: *database0Name
+    documents:
+      - { _id: 1 }
+  - collectionName: *collection1Name
+    databaseName: *database0Name
+    documents: [ ]
+
+tests:
+  - description: "Aggregate with let option"
+    runOnRequirements:
+      - minServerVersion: "5.0"
+    operations:
+      - name: aggregate
+        object: *collection0
+        arguments:
+          pipeline: &pipeline0
+            # $match takes a query expression, so $expr is necessary to utilize
+            # an aggregate expression context and access "let" variables.
+            - $match: { $expr: { $eq: ["$_id", "$$id"] } }
+            - $project: { _id: 0, x: "$$x", y: "$$y", rand: "$$rand" }
+          # Values in "let" must be constant or closed expressions that do not
+          # depend on document values. This test demonstrates a basic constant
+          # value, a value wrapped with $literal (to avoid expression parsing),
+          # and a closed expression (e.g. $rand).
+          let: &let0
+            id: 1
+            x: foo
+            y: { $literal: "$bar" }
+            rand: { $rand: {} }
+        expectResult:
+          - { x: "foo", y: "$bar", rand: { $$type: "double" } }
+    expectEvents:
+      - client: *client0
+        events:
+          - commandStartedEvent:
+              command:
+                aggregate: *collection0Name
+                pipeline: *pipeline0
+                let: *let0
+
+  - description: "Aggregate with let option unsupported (server-side error)"
+    runOnRequirements:
+      - minServerVersion: "2.6.0"
+        maxServerVersion: "4.4.99"
+    operations:
+      - name: aggregate
+        object: *collection0
+        arguments:
+          pipeline: &pipeline1
+            - $match: { _id: 1 }
+          let: &let1
+            x: foo
+        expectError:
+          # Older server versions may not report an error code, but the error
+          # message is consistent between 2.6.x and 4.4.x server versions.
+          errorContains: "unrecognized field 'let'"
+          isClientError: false
+    expectEvents:
+      - client: *client0
+        events:
+          - commandStartedEvent:
+              command:
+                aggregate: *collection0Name
+                pipeline: *pipeline1
+                let: *let1
+
+  - description: "Aggregate to collection with let option"
+    runOnRequirements:
+      - minServerVersion: "5.0"
+        serverless: "forbid"
+    operations:
+      - name: aggregate
+        object: *collection0
+        arguments:
+          pipeline: &pipeline2
+            - $match: { $expr: { $eq: ["$_id", "$$id"] } }
+            - $project: { _id: 1 }
+            - $out: *collection1Name
+          let: &let2
+            id: 1
+    expectEvents:
+      - client: *client0
+        events:
+          - commandStartedEvent:
+              command:
+                aggregate: *collection0Name
+                pipeline: *pipeline2
+                let: *let2
+    outcome:
+      - collectionName: *collection1Name
+        databaseName: *database0Name
+        documents:
+          - { _id: 1 }
+
+  - description: "Aggregate to collection with let option unsupported (server-side error)"
+    runOnRequirements:
+      - minServerVersion: "2.6.0"
+        maxServerVersion: "4.4.99"
+    operations:
+      - name: aggregate
+        object: *collection0
+        arguments:
+          pipeline: *pipeline2
+          let: *let2
+        expectError:
+          errorContains: "unrecognized field 'let'"
+          isClientError: false
+    expectEvents:
+      - client: *client0
+        events:
+          - commandStartedEvent:
+              command:
+                aggregate: *collection0Name
+                pipeline: *pipeline2
+                let: *let2

data/spec/spec_tests/data/crud_unified/aggregate-write-readPreference.yml

@@ -0,0 +1,155 @@
+description: aggregate-write-readPreference
+
+schemaVersion: '1.4'
+
+runOnRequirements:
+  # 3.6+ non-standalone is needed to utilize $readPreference in OP_MSG
+  - minServerVersion: "3.6"
+    topologies: [ replicaset, sharded, load-balanced ]
+
+_yamlAnchors:
+  readConcern: &readConcern
+    level: &readConcernLevel "local"
+  writeConcern: &writeConcern
+    w: &writeConcernW 1
+
+createEntities:
+  - client:
+      id: &client0 client0
+      observeEvents:
+        - commandStartedEvent
+      # Used to test that read and write concerns are still inherited
+      uriOptions:
+        readConcernLevel: *readConcernLevel
+        w: *writeConcernW
+  - database:
+      id: &database0 database0
+      client: *client0
+      databaseName: &database0Name db0
+  - collection:
+      id: &collection0 collection0
+      database: *database0
+      collectionName: &collection0Name coll0
+      collectionOptions:
+        readPreference: &readPreference
+          # secondaryPreferred is specified for compatibility with clusters that
+          # may not have a secondary (e.g. each shard is only a primary).
+          mode: secondaryPreferred
+          # maxStalenessSeconds is specified to ensure that drivers forward the
+          # read preference to mongos or a load balancer. That would not be the
+          # case with only secondaryPreferred.
+          maxStalenessSeconds: 600
+  - collection:
+      id: &collection1 collection1
+      database: *database0
+      collectionName: &collection1Name coll1
+
+initialData:
+  - collectionName: *collection0Name
+    databaseName: *database0Name
+    documents:
+      - { _id: 1, x: 11 }
+      - { _id: 2, x: 22 }
+      - { _id: 3, x: 33 }
+  - collectionName: *collection1Name
+    databaseName: *database0Name
+    documents: []
+
+tests:
+  - description: "Aggregate with $out includes read preference for 5.0+ server"
+    runOnRequirements:
+      - minServerVersion: "5.0"
+        serverless: "forbid"
+    operations:
+      - object: *collection0
+        name: aggregate
+        arguments:
+          pipeline: &outPipeline
+            - { $match: { _id: { $gt: 1 } } }
+            - { $sort: { x: 1 } }
+            - { $out: *collection1Name }
+    expectEvents:
+      - client: *client0
+        events:
+          - commandStartedEvent:
+              command:
+                aggregate: *collection0Name
+                pipeline: *outPipeline
+                $readPreference: *readPreference
+                readConcern: *readConcern
+                writeConcern: *writeConcern
+    outcome: &outcome
+      - collectionName: *collection1Name
+        databaseName: *database0Name
+        documents:
+          - { _id: 2, x: 22 }
+          - { _id: 3, x: 33 }
+
+  - description: "Aggregate with $out omits read preference for pre-5.0 server"
+    runOnRequirements:
+      # MongoDB 4.2 introduced support for read concerns and write stages.
+      # Pre-4.2 servers may allow a "local" read concern anyway, but some
+      # drivers may avoid inheriting a client-level read concern for pre-4.2.
+      - minServerVersion: "4.2"
+        maxServerVersion: "4.4.99"
+        serverless: "forbid"
+    operations:
+      - object: *collection0
+        name: aggregate
+        arguments:
+          pipeline: *outPipeline
+    expectEvents:
+      - client: *client0
+        events:
+          - commandStartedEvent:
+              command:
+                aggregate: *collection0Name
+                pipeline: *outPipeline
+                $readPreference: { $$exists: false }
+                readConcern: *readConcern
+                writeConcern: *writeConcern
+    outcome: *outcome
+
+  - description: "Aggregate with $merge includes read preference for 5.0+ server"
+    runOnRequirements:
+      - minServerVersion: "5.0"
+    operations:
+      - object: *collection0
+        name: aggregate
+        arguments:
+          pipeline: &mergePipeline
+            - { $match: { _id: { $gt: 1 } } }
+            - { $sort: { x: 1 } }
+            - { $merge: { into: *collection1Name } }
+    expectEvents:
+      - client: *client0
+        events:
+          - commandStartedEvent:
+              command:
+                aggregate: *collection0Name
+                pipeline: *mergePipeline
+                $readPreference: *readPreference
+                readConcern: *readConcern
+                writeConcern: *writeConcern
+    outcome: *outcome
+
+  - description: "Aggregate with $merge omits read preference for pre-5.0 server"
+    runOnRequirements:
+      - minServerVersion: "4.2"
+        maxServerVersion: "4.4.99"
+    operations:
+      - object: *collection0
+        name: aggregate
+        arguments:
+          pipeline: *mergePipeline
+    expectEvents:
+      - client: *client0
+        events:
+          - commandStartedEvent:
+              command:
+                aggregate: *collection0Name
+                pipeline: *mergePipeline
+                $readPreference: { $$exists: false }
+                readConcern: *readConcern
+                writeConcern: *writeConcern
+    outcome: *outcome

data/spec/spec_tests/data/crud_unified/db-aggregate-write-readPreference.yml

@@ -0,0 +1,151 @@
+description: db-aggregate-write-readPreference
+
+schemaVersion: '1.4'
+
+runOnRequirements:
+  # 3.6+ non-standalone is needed to utilize $readPreference in OP_MSG.
+  # Serverless does not support $listLocalSessions and $currentOp stages, and
+  # mongos does not allow combining them with $out or $merge.
+  - minServerVersion: "3.6"
+    topologies: [ replicaset ]
+    serverless: forbid
+
+_yamlAnchors:
+  readConcern: &readConcern
+    level: &readConcernLevel "local"
+  writeConcern: &writeConcern
+    w: &writeConcernW 1
+
+createEntities:
+  - client:
+      id: &client0 client0
+      observeEvents:
+        - commandStartedEvent
+      # Used to test that read and write concerns are still inherited
+      uriOptions:
+        readConcernLevel: *readConcernLevel
+        w: *writeConcernW
+  - database:
+      id: &database0 database0
+      client: *client0
+      databaseName: &database0Name db0
+      databaseOptions:
+        readPreference: &readPreference
+          # secondaryPreferred is specified for compatibility with clusters that
+          # may not have a secondary (e.g. each shard is only a primary).
+          mode: secondaryPreferred
+          # maxStalenessSeconds is specified to ensure that drivers forward the
+          # read preference to mongos or a load balancer. That would not be the
+          # case with only secondaryPreferred.
+          maxStalenessSeconds: 600
+  - collection:
+      id: &collection0 collection0
+      database: *database0
+      collectionName: &collection0Name coll0
+
+initialData:
+  - collectionName: *collection0Name
+    databaseName: *database0Name
+    documents: []
+
+tests:
+  - description: "Database-level aggregate with $out includes read preference for 5.0+ server"
+    runOnRequirements:
+      - minServerVersion: "5.0"
+        serverless: "forbid"
+    operations:
+      - object: *database0
+        name: aggregate
+        arguments:
+          pipeline: &outPipeline
+            - { $listLocalSessions: {} }
+            - { $limit: 1 }
+            - { $addFields: { _id: 1 } }
+            - { $project: { _id: 1 } }
+            - { $out: *collection0Name }
+    expectEvents:
+      - client: *client0
+        events:
+          - commandStartedEvent:
+              command:
+                aggregate: 1
+                pipeline: *outPipeline
+                $readPreference: *readPreference
+                readConcern: *readConcern
+                writeConcern: *writeConcern
+    outcome: &outcome
+      - collectionName: *collection0Name
+        databaseName: *database0Name
+        documents:
+          - { _id: 1 }
+
+  - description: "Database-level aggregate with $out omits read preference for pre-5.0 server"
+    runOnRequirements:
+      # MongoDB 4.2 introduced support for read concerns and write stages.
+      # Pre-4.2 servers may allow a "local" read concern anyway, but some
+      # drivers may avoid inheriting a client-level read concern for pre-4.2.
+      - minServerVersion: "4.2"
+        maxServerVersion: "4.4.99"
+        serverless: "forbid"
+    operations:
+      - object: *database0
+        name: aggregate
+        arguments:
+          pipeline: *outPipeline
+    expectEvents:
+      - client: *client0
+        events:
+          - commandStartedEvent:
+              command:
+                aggregate: 1
+                pipeline: *outPipeline
+                $readPreference: { $$exists: false }
+                readConcern: *readConcern
+                writeConcern: *writeConcern
+    outcome: *outcome
+
+  - description: "Database-level aggregate with $merge includes read preference for 5.0+ server"
+    runOnRequirements:
+      - minServerVersion: "5.0"
+    operations:
+      - object: *database0
+        name: aggregate
+        arguments:
+          pipeline: &mergePipeline
+            - { $listLocalSessions: {} }
+            - { $limit: 1 }
+            - { $addFields: { _id: 1 } }
+            - { $project: { _id: 1 } }
+            - { $merge: { into: *collection0Name } }
+    expectEvents:
+      - client: *client0
+        events:
+          - commandStartedEvent:
+              command:
+                aggregate: 1
+                pipeline: *mergePipeline
+                $readPreference: *readPreference
+                readConcern: *readConcern
+                writeConcern: *writeConcern
+    outcome: *outcome
+
+  - description: "Database-level aggregate with $merge omits read preference for pre-5.0 server"
+    runOnRequirements:
+      - minServerVersion: "4.2"
+        maxServerVersion: "4.4.99"
+    operations:
+      - object: *database0
+        name: aggregate
+        arguments:
+          pipeline: *mergePipeline
+    expectEvents:
+      - client: *client0
+        events:
+          - commandStartedEvent:
+              command:
+                aggregate: 1
+                pipeline: *mergePipeline
+                $readPreference: { $$exists: false }
+                readConcern: *readConcern
+                writeConcern: *writeConcern
+    outcome: *outcome