mongoid 9.0.11 → 9.0.12

data/spec/integration/encryption_spec.rb

@@ -34,19 +34,24 @@ describe 'Encryption' do
   around do |example|
     Mongoid.default_client[Crypt::Patient.collection_name].drop
     Mongoid.default_client[Crypt::Car.collection_name].drop
+    Mongoid.default_client[Crypt::Folder.collection_name].drop
     existing_key_id = Crypt::Patient.encrypt_metadata[:key_id]
     Crypt::Patient.set_key_id(data_key_id)
     Crypt::Car.set_key_id(data_key_id)
+    Crypt::Note.set_key_id(data_key_id)
     Mongoid::Config.send(:clients=, config)
     Mongoid::Clients.with_name(:key_vault)[key_vault_collection].drop
     Crypt::Patient.store_in(client: :encrypted)
     Crypt::Car.store_in(client: :encrypted, database: Crypt::Car.storage_options[:database])
+    Crypt::Folder.store_in(client: :encrypted)
 
     example.run
 
     Crypt::Patient.reset_storage_options!
+    Crypt::Folder.reset_storage_options!
     Crypt::Patient.set_key_id(existing_key_id)
     Crypt::Car.set_key_id(existing_key_id)
+    Crypt::Note.set_key_id(existing_key_id)
   end
 
   it 'encrypts and decrypts fields' do
@@ -86,6 +91,17 @@ describe 'Encryption' do
     end
   end
 
+  # Nothing about the parent document says it needs a schema: it has no
+  # encrypted field of its own and no encrypt_with. Its collection still needs
+  # one, or the embedded field goes out in the clear.
+  it 'stores an embedded field encrypted when the parent has no encrypted field' do
+    folder = Crypt::Folder.create!(note: Crypt::Note.new(text: 'SECRET'))
+    unencrypted_client[Crypt::Folder.collection.name].find(_id: folder.id).first.tap do |doc|
+      expect(doc['note']['text']).to be_a(BSON::Binary)
+      expect(doc['note']['text'].type).to eq(:ciphertext)
+    end
+  end
+
   it 'stores data encrypted in the non-default database' do
     car = Crypt::Car.create!(vin: 'VA1234')
     unencrypted_client
@@ -96,5 +112,138 @@ describe 'Encryption' do
     end
   end
 
+  # The encryption schema map is keyed by namespace and built once, when the
+  # client is constructed. Whenever the namespace an encrypted model actually
+  # writes to is not in that map, the driver encrypts nothing and reports no
+  # error, so the field lands in the clear. Mongoid fails closed instead.
+  describe 'when the target namespace is not in the encryption schema map' do
+    let(:scratch_databases) do
+      %w[vehicles_dynamic vehicles_tenant_b vehicles_tenant_c tenant_a]
+    end
+
+    around do |example|
+      original_options = {
+        Crypt::DynamicCar => Crypt::DynamicCar.storage_options,
+        Crypt::TenantCar => Crypt::TenantCar.storage_options
+      }
+      existing_key_ids = original_options.keys.to_h { |model| [ model, model.encrypt_metadata[:key_id] ] }
+      original_options.each_key do |model|
+        model.set_key_id(data_key_id)
+        model.store_in(client: :encrypted)
+      end
+      clean_scratch_data
+      # The schema map is generated when the client is built, so the client has
+      # to be built after the storage options above are in place.
+      Mongoid::Clients.clear
+
+      example.run
+
+      clean_scratch_data
+      original_options.each { |model, options| model.storage_options = options }
+      existing_key_ids.each { |model, key_id| model.set_key_id(key_id) }
+      Mongoid::Clients.clear
+    end
+
+    def clean_scratch_data
+      scratch_databases.each { |database| unencrypted_client.use(database).database.drop }
+      # Crypt::Car writes into the shared 'vehicles' database, which other
+      # examples rely on, so remove only the documents these examples create.
+      unencrypted_client
+        .use(Crypt::Car.storage_options[:database])[Crypt::Car.collection_name.to_s]
+        .delete_many(vin: { '$in' => %w[CLIENT-1 CLIENT-2] })
+    end
+
+    # Reads with a client that has no automatic encryption, to see what really
+    # landed on disk.
+    def documents_with_plaintext_vin(database, collection, vin)
+      unencrypted_client.use(database)[collection].find(vin: vin).to_a
+    end
+
+    context 'when the database name is a callable' do
+      it 'does not store the field in plaintext', :aggregate_failures do
+        expect { Crypt::DynamicCar.create!(vin: 'DYNAMIC-1') }
+          .to raise_error(Mongoid::Errors::NoEncryptionSchema)
+
+        expect(
+          documents_with_plaintext_vin('vehicles_dynamic', Crypt::DynamicCar.collection_name.to_s, 'DYNAMIC-1')
+        ).to be_empty
+      end
+    end
+
+    context 'when the database name is a callable resolved per tenant' do
+      around do |example|
+        Thread.current[:tenant_database] = 'tenant_a'
+        example.run
+        Thread.current[:tenant_database] = nil
+      end
+
+      it 'does not store the field in plaintext', :aggregate_failures do
+        expect { Crypt::TenantCar.create!(vin: 'TENANT-1') }
+          .to raise_error(Mongoid::Errors::NoEncryptionSchema)
+
+        expect(
+          documents_with_plaintext_vin('tenant_a', Crypt::TenantCar.collection_name.to_s, 'TENANT-1')
+        ).to be_empty
+      end
+    end
+
+    context 'when the database is overridden for the block' do
+      it 'does not store the field in plaintext', :aggregate_failures do
+        expect { Crypt::Car.with(database: 'vehicles_tenant_b') { |car| car.create!(vin: 'BLOCK-1') } }
+          .to raise_error(Mongoid::Errors::NoEncryptionSchema)
+
+        expect(
+          documents_with_plaintext_vin('vehicles_tenant_b', Crypt::Car.collection_name.to_s, 'BLOCK-1')
+        ).to be_empty
+      end
+    end
+
+    context 'when the database is overridden globally' do
+      persistence_context_override :database, 'vehicles_tenant_c'
 
+      it 'does not store the field in plaintext', :aggregate_failures do
+        expect { Crypt::Car.create!(vin: 'GLOBAL-1') }
+          .to raise_error(Mongoid::Errors::NoEncryptionSchema)
+
+        expect(
+          documents_with_plaintext_vin('vehicles_tenant_c', Crypt::Car.collection_name.to_s, 'GLOBAL-1')
+        ).to be_empty
+      end
+    end
+
+    context 'when the model is routed to a client without automatic encryption' do
+      it 'does not store the field in plaintext', :aggregate_failures do
+        expect { Crypt::Car.with(client: :default) { |car| car.create!(vin: 'CLIENT-1') } }
+          .to raise_error(Mongoid::Errors::NoEncryptionSchema)
+
+        expect(
+          documents_with_plaintext_vin('vehicles', Crypt::Car.collection_name.to_s, 'CLIENT-1')
+        ).to be_empty
+      end
+    end
+
+    context 'when the parent has no encrypted field of its own' do
+      it 'does not store the embedded field in plaintext', :aggregate_failures do
+        expect { Crypt::Folder.with(client: :default) { |folder| folder.create!(note: Crypt::Note.new(text: 'EMBEDDED-1')) } }
+          .to raise_error(Mongoid::Errors::NoEncryptionSchema)
+
+        expect(
+          unencrypted_client[Crypt::Folder.collection_name.to_s].find('note.text' => 'EMBEDDED-1').to_a
+        ).to be_empty
+      end
+    end
+
+    context 'when the client is overridden globally to one without automatic encryption' do
+      persistence_context_override :client, :default
+
+      it 'does not store the field in plaintext', :aggregate_failures do
+        expect { Crypt::Car.create!(vin: 'CLIENT-2') }
+          .to raise_error(Mongoid::Errors::NoEncryptionSchema)
+
+        expect(
+          documents_with_plaintext_vin('vehicles', Crypt::Car.collection_name.to_s, 'CLIENT-2')
+        ).to be_empty
+      end
+    end
+  end
 end

data/spec/integration/matcher_operator_data/regex.yml

@@ -135,6 +135,27 @@
       $regex: bar
   matches: false
 
+# Object#=~ is gone as of Ruby 3.2, so an element that cannot be matched
+# against has to be skipped rather than passed to =~.
+- name: array field value - element that cannot be matched against
+  document:
+    title:
+      - 42
+      - foo
+  query:
+    title:
+      $regex: foo
+  matches: true
+
+- name: array field value - no element can be matched against
+  document:
+    title:
+      - 42
+  query:
+    title:
+      $regex: foo
+  matches: false
+
 - name: true field value
   document:
     title: true

data/spec/integration/matcher_regexp_timeout_spec.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require 'benchmark'
+require 'spec_helper'
+
+describe 'in-memory regexp time limit' do
+  # See the comment in spec/mongoid/matcher/regexp_budget_spec.rb: costly to
+  # match but bounded on every supported Ruby. A pattern relying on nested
+  # quantifiers would hang before Ruby 3.2, where the examples that have no
+  # interrupt available could not stop it.
+  let(:slow_pattern) { "(?:#{(1..300).map { |i| "a#{i}" }.join('|')})Z" }
+  let(:slow_street) { 'a' * 5_000 }
+
+  let(:person) do
+    # Address derives its _id from the street, so the streets have to differ or
+    # the embedded association collapses to a single document.
+    Person.new(addresses: Array.new(30) { |i| Address.new(street: "#{slow_street}#{i}") })
+  end
+
+  let(:one_address) { Person.new(addresses: [ Address.new(street: slow_street) ]) }
+
+  # What one match of the fixture costs on the machine running the suite. It
+  # varies by more than an order of magnitude between implementations - around
+  # 6ms on MRI, 110ms on JRuby - so the examples below calibrate against a
+  # measurement instead of hard-coding a threshold that only holds on one.
+  let(:single_match_cost) do
+    regexp = Regexp.new(slow_pattern)
+    3.times { slow_street =~ regexp }
+    Benchmark.realtime { slow_street =~ regexp }
+  end
+
+  # Comfortably more than one match, comfortably less than twenty. The examples
+  # only need the ordering to hold, so the margin either side is wide.
+  let(:calibrated_limit) { single_match_cost * 5 }
+
+  context 'when the limit is disabled' do
+    config_override :in_memory_regexp_time_limit, nil
+
+    it 'evaluates the query without a bound' do
+      expect(person.addresses.where(street: { '$regex' => slow_pattern }).to_a).to eq([])
+    end
+  end
+
+  context 'when a limit is configured' do
+    config_override :in_memory_regexp_time_limit, nil
+
+    before { Mongoid::Config.in_memory_regexp_time_limit = calibrated_limit }
+
+    it 'leaves ordinary queries alone' do
+      person = Person.new(addresses: [ Address.new(street: 'Clarkson') ])
+      expect(person.addresses.where(street: { '$regex' => '\AClark' }).to_a.size).to eq(1)
+    end
+
+    it 'does not raise for a single document' do
+      # The counterpart to the example below, and what makes it meaningful: one
+      # document stays under the limit, so a scan that raises can only have got
+      # there by accumulating cost across documents.
+      expect do
+        one_address.addresses.where(street: { '$regex' => slow_pattern }).to_a
+      end.not_to raise_error
+    end
+
+    it 'raises for a scan over many documents under that same limit' do
+      expect do
+        person.addresses.where(street: { '$regex' => slow_pattern }).to_a
+      end.to raise_error(Mongoid::Errors::InMemoryRegexpTimeout)
+    end
+
+    it 'raises when $in multiplies the cost within a single document' do
+      conditions = Array.new(20) { BSON::Regexp::Raw.new(slow_pattern) }
+
+      expect do
+        one_address.addresses.where(street: { '$in' => conditions }).to_a
+      end.to raise_error(Mongoid::Errors::InMemoryRegexpTimeout)
+    end
+
+    it 'raises through the public _matches? API' do
+      # 20 conditions rather than one, because a single match is deliberately
+      # kept well under the limit by the calibration above.
+      conditions = Array.new(20) { BSON::Regexp::Raw.new(slow_pattern) }
+
+      expect do
+        person.addresses.first._matches?('street' => { '$in' => conditions })
+      end.to raise_error(Mongoid::Errors::InMemoryRegexpTimeout)
+    end
+  end
+
+  context 'when the pattern is pathological' do
+    config_override :in_memory_regexp_time_limit, 0.2
+
+    # The ticket's Case 1, verbatim. It backtracks exponentially and is the
+    # reason the guard exists, so every entry point that evaluates a selector
+    # in memory has to bound it. The subject is kept short enough that the
+    # match still finishes on its own in a few seconds, so an entry point that
+    # fails to bound it shows up as a failing example rather than a hung run.
+    let(:evil_pattern) { '^(a+)+\1?$' }
+    let(:evil_subject) { "#{'a' * 28}X" }
+
+    it 'bounds a scan over an embedded association' do
+      person = Person.new(addresses: [ Address.new(street: evil_subject) ])
+
+      expect do
+        person.addresses.where(street: { '$regex' => evil_pattern }).to_a
+      end.to raise_error(Mongoid::Errors::InMemoryRegexpTimeout)
+    end
+
+    it 'bounds the public _matches? API' do
+      expect do
+        Address.new(street: evil_subject)._matches?('street' => { '$regex' => evil_pattern })
+      end.to raise_error(Mongoid::Errors::InMemoryRegexpTimeout)
+    end
+
+    it 'bounds removal from a referenced association' do
+      owner = Person.create!
+      owner.posts.create!(title: evil_subject)
+
+      expect do
+        owner.posts.delete_all(title: { '$regex' => evil_pattern })
+      end.to raise_error(Mongoid::Errors::InMemoryRegexpTimeout)
+    end
+
+    it 'deletes nothing when removal is bounded' do
+      # The scan runs before the delete, so the error means what a caller would
+      # take it to mean: the documents are still there.
+      owner = Person.create!
+      owner.posts.create!(title: evil_subject)
+
+      expect do
+        owner.posts.delete_all(title: { '$regex' => evil_pattern })
+      end.to raise_error(Mongoid::Errors::InMemoryRegexpTimeout)
+
+      expect(Post.count).to eq(1)
+      expect(owner.reload.posts.size).to eq(1)
+    end
+  end
+
+  context 'when the association being removed from has not been loaded' do
+    config_override :in_memory_regexp_time_limit, 0.2
+
+    let!(:owner) do
+      person = Person.create!
+      person.posts.create!(title: 'Testing')
+      Person.find(person._id)
+    end
+
+    # The query that loads the association, which removal has to run before it
+    # has anything to match against.
+    let(:load_query) { owner.posts._target._unloaded }
+
+    # A stall standing in for a slow network. It is longer than the limit, and
+    # none of it is time spent on regular expressions.
+    before do
+      allow(load_query).to receive(:each).and_wrap_original do |original, *args, &block|
+        sleep(0.3)
+        original.call(*args, &block)
+      end
+    end
+
+    it 'does not load it, so no query can be counted against the limit' do
+      # On the Timeout path a deadline covering the fetch failed a slow query
+      # with an error about regular expressions, and the asynchronous exception
+      # could land inside the driver's socket read, leaving the connection with
+      # unconsumed bytes. Nothing is fetched now: the scan sees only what the
+      # association already holds.
+      stub_const('Mongoid::Matcher::RegexpBudget::PER_REGEXP_TIMEOUT', false)
+
+      expect do
+        owner.posts.delete_all(title: { '$regex' => '\Azzz' })
+      end.not_to raise_error
+
+      expect(load_query).not_to have_received(:each)
+    end
+
+    it 'runs no pattern against documents it never loaded' do
+      # The counterpart to the referenced-association examples below, where the
+      # documents are in memory and the pattern is run against every one of
+      # them. Here there is nothing to run it against, and the server applies
+      # the same selector for the delete, so nothing is missed by not looking.
+      expect(Mongoid::Matcher::RegexpBudget).not_to receive(:match?)
+
+      owner.posts.delete_all(title: { '$regex' => '\Azzz' })
+    end
+
+    it 'still removes the documents that match' do
+      expect(owner.posts.delete_all(title: { '$regex' => '\ATest' })).to eq(1)
+      expect(Post.count).to eq(0)
+      expect(owner.posts.size).to eq(0)
+    end
+  end
+
+  context 'when removing documents from a referenced association' do
+    config_override :in_memory_regexp_time_limit, nil
+
+    let!(:owner) { Person.create! }
+
+    before do
+      30.times { owner.posts.create!(title: slow_street) }
+      Mongoid::Config.in_memory_regexp_time_limit = calibrated_limit
+    end
+
+    it 'raises rather than scanning every loaded document' do
+      expect do
+        owner.posts.delete_all(title: { '$regex' => slow_pattern })
+      end.to raise_error(Mongoid::Errors::InMemoryRegexpTimeout)
+    end
+
+    it 'leaves the documents in place' do
+      expect do
+        owner.posts.delete_all(title: { '$regex' => slow_pattern })
+      end.to raise_error(Mongoid::Errors::InMemoryRegexpTimeout)
+
+      expect(Post.count).to eq(30)
+    end
+  end
+end

data/spec/integration/query_operator_guard_spec.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe 'query operator injection guard' do
+  let(:js) { 'this.name == "admin"' }
+
+  let(:nested_function) do
+    { '$expr' => { '$function' => { 'body' => 'function() { return true; }', 'args' => [], 'lang' => 'js' } } }
+  end
+
+  let(:band) { Band.create!(name: 'Depeche Mode') }
+
+  context 'with default configuration' do
+    it 'rejects unsafe operators without any opt-out' do
+      expect(Mongoid.allow_unsafe_query_operators).to be false
+    end
+
+    context 'when querying with where' do
+      it 'rejects a top-level $where' do
+        expect { Band.where('$where' => js).first }.to raise_error(Mongoid::Errors::InvalidQuery)
+      end
+
+      it 'rejects a $function nested in $expr' do
+        expect { Band.where(nested_function).first }.to raise_error(Mongoid::Errors::InvalidQuery)
+      end
+
+      it 'permits an ordinary query' do
+        expect(Band.where(name: band.name).first).to eq(band)
+      end
+    end
+
+    context 'when querying with find_by' do
+      it 'rejects a top-level $where' do
+        expect { Band.find_by('$where' => js) }.to raise_error(Mongoid::Errors::InvalidQuery)
+      end
+
+      it 'rejects a $function nested in $expr' do
+        expect { Band.find_by(nested_function) }.to raise_error(Mongoid::Errors::InvalidQuery)
+      end
+    end
+
+    context 'when querying with find_or_create_by' do
+      it 'rejects a top-level $where' do
+        expect { Band.find_or_create_by('$where' => js) }.to raise_error(Mongoid::Errors::InvalidQuery)
+      end
+    end
+
+    context 'when querying with find_or_initialize_by' do
+      it 'rejects a top-level $where' do
+        expect { Band.find_or_initialize_by('$where' => js) }.to raise_error(Mongoid::Errors::InvalidQuery)
+      end
+    end
+
+    context 'when querying an association with find_or_create_by' do
+      it 'rejects a top-level $where' do
+        expect do
+          band.records.find_or_create_by('$where' => js)
+        end.to raise_error(Mongoid::Errors::InvalidQuery)
+      end
+    end
+
+    context 'when querying with a logical operator' do
+      it 'rejects a $where smuggled through or' do
+        expect { Band.or('$where' => js).first }.to raise_error(Mongoid::Errors::InvalidQuery)
+      end
+
+      it 'rejects a $where smuggled through any_of' do
+        expect { Band.any_of('$where' => js).first }.to raise_error(Mongoid::Errors::InvalidQuery)
+      end
+    end
+  end
+
+  context 'when allow_unsafe_query_operators is true' do
+    config_override :allow_unsafe_query_operators, true
+
+    it 'permits a top-level $where' do
+      expect(Band.where('$where' => "this.name == '#{band.name}'").first).to eq(band)
+    end
+
+    it 'permits a $function nested in $expr' do
+      expect { Band.where(nested_function).first }.not_to raise_error
+    end
+
+    it 'permits a $where smuggled through or' do
+      expect { Band.or('$where' => js).first }.not_to raise_error
+    end
+  end
+end

data/spec/mongoid/association/referenced/has_and_belongs_to_many/proxy_spec.rb

@@ -2267,6 +2267,13 @@ describe Mongoid::Association::Referenced::HasAndBelongsToMany::Proxy do
     end
   end
 
+  # Whether the conditions carry a pattern decides which branch of remove_all
+  # runs, so anything that has to behave the same on both is exercised twice.
+  conditions_variants = [
+    [ 'no regular expression', nil ],
+    [ 'a regular expression', { name: /Test/ } ]
+  ].freeze
+
   %i[ delete_all destroy_all ].each do |method|
     describe "\##{method}" do
       context 'when the relation is not polymorphic' do
@@ -2338,6 +2345,47 @@ describe Mongoid::Association::Referenced::HasAndBelongsToMany::Proxy do
             expect(person.preferences).to eq([])
           end
         end
+
+        context 'when the base holds the foreign keys' do
+          # unbind_one pulls ids out of the base's foreign key array and marks
+          # the base dirty, so which documents get unbound is visible here.
+          # Only what the association already held is unbound, and that has to
+          # hold whether or not the conditions carry a regular expression:
+          # scanning under a regexp budget loads the association first, and
+          # that load must not turn into an unbind of its own. See the note in
+          # HasMany::Proxy#remove_all_bounded.
+          let!(:person) do
+            Person.create!.tap do |base|
+              base.preferences.create!(name: 'Testing')
+              base.preferences.create!(name: 'Test')
+            end
+          end
+
+          let(:ids) { person.preference_ids.dup }
+
+          conditions_variants.each do |description, conditions|
+            context "when the conditions carry #{description}" do
+              context 'when the association is already in memory' do
+                it 'unbinds the deleted documents' do
+                  person.preferences.send(method, conditions)
+                  expect(person.preference_ids).to eq([])
+                  expect(person).to be_changed
+                end
+              end
+
+              context 'when the association has not been loaded' do
+                let(:cold) { Person.find(person._id) }
+
+                it 'leaves the foreign keys alone' do
+                  expect(ids.length).to eq(2)
+                  cold.preferences.send(method, conditions)
+                  expect(cold.preference_ids).to eq(ids)
+                  expect(cold).not_to be_changed
+                end
+              end
+            end
+          end
+        end
       end
     end
   end