mongoid 8.0.11 → 8.0.13

data/spec/integration/matcher_operator_data/regex.yml

@@ -135,6 +135,27 @@
       $regex: bar
   matches: false
 
+# Object#=~ is gone as of Ruby 3.2, so an element that cannot be matched
+# against has to be skipped rather than passed to =~.
+- name: array field value - element that cannot be matched against
+  document:
+    title:
+      - 42
+      - foo
+  query:
+    title:
+      $regex: foo
+  matches: true
+
+- name: array field value - no element can be matched against
+  document:
+    title:
+      - 42
+  query:
+    title:
+      $regex: foo
+  matches: false
+
 - name: true field value
   document:
     title: true

data/spec/integration/matcher_regexp_timeout_spec.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require 'benchmark'
+require 'spec_helper'
+
+describe 'in-memory regexp time limit' do
+  # See the comment in spec/mongoid/matcher/regexp_budget_spec.rb: costly to
+  # match but bounded on every supported Ruby. A pattern relying on nested
+  # quantifiers would hang before Ruby 3.2, where the examples that have no
+  # interrupt available could not stop it.
+  let(:slow_pattern) { "(?:#{(1..300).map { |i| "a#{i}" }.join('|')})Z" }
+  let(:slow_street) { 'a' * 5_000 }
+
+  let(:person) do
+    # Address derives its _id from the street, so the streets have to differ or
+    # the embedded association collapses to a single document.
+    Person.new(addresses: Array.new(30) { |i| Address.new(street: "#{slow_street}#{i}") })
+  end
+
+  let(:one_address) { Person.new(addresses: [ Address.new(street: slow_street) ]) }
+
+  # What one match of the fixture costs on the machine running the suite. It
+  # varies by more than an order of magnitude between implementations - around
+  # 6ms on MRI, 110ms on JRuby - so the examples below calibrate against a
+  # measurement instead of hard-coding a threshold that only holds on one.
+  let(:single_match_cost) do
+    regexp = Regexp.new(slow_pattern)
+    3.times { slow_street =~ regexp }
+    Benchmark.realtime { slow_street =~ regexp }
+  end
+
+  # Comfortably more than one match, comfortably less than twenty. The examples
+  # only need the ordering to hold, so the margin either side is wide.
+  let(:calibrated_limit) { single_match_cost * 5 }
+
+  context 'when the limit is disabled' do
+    config_override :in_memory_regexp_time_limit, nil
+
+    it 'evaluates the query without a bound' do
+      expect(person.addresses.where(street: { '$regex' => slow_pattern }).to_a).to eq([])
+    end
+  end
+
+  context 'when a limit is configured' do
+    config_override :in_memory_regexp_time_limit, nil
+
+    before { Mongoid::Config.in_memory_regexp_time_limit = calibrated_limit }
+
+    it 'leaves ordinary queries alone' do
+      person = Person.new(addresses: [ Address.new(street: 'Clarkson') ])
+      expect(person.addresses.where(street: { '$regex' => '\AClark' }).to_a.size).to eq(1)
+    end
+
+    it 'does not raise for a single document' do
+      # The counterpart to the example below, and what makes it meaningful: one
+      # document stays under the limit, so a scan that raises can only have got
+      # there by accumulating cost across documents.
+      expect do
+        one_address.addresses.where(street: { '$regex' => slow_pattern }).to_a
+      end.not_to raise_error
+    end
+
+    it 'raises for a scan over many documents under that same limit' do
+      expect do
+        person.addresses.where(street: { '$regex' => slow_pattern }).to_a
+      end.to raise_error(Mongoid::Errors::InMemoryRegexpTimeout)
+    end
+
+    it 'raises when $in multiplies the cost within a single document' do
+      conditions = Array.new(20) { BSON::Regexp::Raw.new(slow_pattern) }
+
+      expect do
+        one_address.addresses.where(street: { '$in' => conditions }).to_a
+      end.to raise_error(Mongoid::Errors::InMemoryRegexpTimeout)
+    end
+
+    it 'raises through the public _matches? API' do
+      # 20 conditions rather than one, because a single match is deliberately
+      # kept well under the limit by the calibration above.
+      conditions = Array.new(20) { BSON::Regexp::Raw.new(slow_pattern) }
+
+      expect do
+        person.addresses.first._matches?('street' => { '$in' => conditions })
+      end.to raise_error(Mongoid::Errors::InMemoryRegexpTimeout)
+    end
+  end
+
+  context 'when the pattern is pathological' do
+    config_override :in_memory_regexp_time_limit, 0.2
+
+    # The ticket's Case 1, verbatim. It backtracks exponentially and is the
+    # reason the guard exists, so every entry point that evaluates a selector
+    # in memory has to bound it. The subject is kept short enough that the
+    # match still finishes on its own in a few seconds, so an entry point that
+    # fails to bound it shows up as a failing example rather than a hung run.
+    let(:evil_pattern) { '^(a+)+\1?$' }
+    let(:evil_subject) { "#{'a' * 28}X" }
+
+    it 'bounds a scan over an embedded association' do
+      person = Person.new(addresses: [ Address.new(street: evil_subject) ])
+
+      expect do
+        person.addresses.where(street: { '$regex' => evil_pattern }).to_a
+      end.to raise_error(Mongoid::Errors::InMemoryRegexpTimeout)
+    end
+
+    it 'bounds the public _matches? API' do
+      expect do
+        Address.new(street: evil_subject)._matches?('street' => { '$regex' => evil_pattern })
+      end.to raise_error(Mongoid::Errors::InMemoryRegexpTimeout)
+    end
+
+    it 'bounds removal from a referenced association' do
+      owner = Person.create!
+      owner.posts.create!(title: evil_subject)
+
+      expect do
+        owner.posts.delete_all(title: { '$regex' => evil_pattern })
+      end.to raise_error(Mongoid::Errors::InMemoryRegexpTimeout)
+    end
+
+    it 'deletes nothing when removal is bounded' do
+      # The scan runs before the delete, so the error means what a caller would
+      # take it to mean: the documents are still there.
+      owner = Person.create!
+      owner.posts.create!(title: evil_subject)
+
+      expect do
+        owner.posts.delete_all(title: { '$regex' => evil_pattern })
+      end.to raise_error(Mongoid::Errors::InMemoryRegexpTimeout)
+
+      expect(Post.count).to eq(1)
+      expect(owner.reload.posts.size).to eq(1)
+    end
+  end
+
+  context 'when the association being removed from has not been loaded' do
+    config_override :in_memory_regexp_time_limit, 0.2
+
+    let!(:owner) do
+      person = Person.create!
+      person.posts.create!(title: 'Testing')
+      Person.find(person._id)
+    end
+
+    # The query that loads the association, which removal has to run before it
+    # has anything to match against.
+    let(:load_query) { owner.posts._target._unloaded }
+
+    # A stall standing in for a slow network. It is longer than the limit, and
+    # none of it is time spent on regular expressions.
+    before do
+      allow(load_query).to receive(:each).and_wrap_original do |original, *args, &block|
+        sleep(0.3)
+        original.call(*args, &block)
+      end
+    end
+
+    it 'does not load it, so no query can be counted against the limit' do
+      # On the Timeout path a deadline covering the fetch failed a slow query
+      # with an error about regular expressions, and the asynchronous exception
+      # could land inside the driver's socket read, leaving the connection with
+      # unconsumed bytes. Nothing is fetched now: the scan sees only what the
+      # association already holds.
+      stub_const('Mongoid::Matcher::RegexpBudget::PER_REGEXP_TIMEOUT', false)
+
+      expect do
+        owner.posts.delete_all(title: { '$regex' => '\Azzz' })
+      end.not_to raise_error
+
+      expect(load_query).not_to have_received(:each)
+    end
+
+    it 'runs no pattern against documents it never loaded' do
+      # The counterpart to the referenced-association examples below, where the
+      # documents are in memory and the pattern is run against every one of
+      # them. Here there is nothing to run it against, and the server applies
+      # the same selector for the delete, so nothing is missed by not looking.
+      expect(Mongoid::Matcher::RegexpBudget).not_to receive(:match?)
+
+      owner.posts.delete_all(title: { '$regex' => '\Azzz' })
+    end
+
+    it 'still removes the documents that match' do
+      expect(owner.posts.delete_all(title: { '$regex' => '\ATest' })).to eq(1)
+      expect(Post.count).to eq(0)
+      expect(owner.posts.size).to eq(0)
+    end
+  end
+
+  context 'when removing documents from a referenced association' do
+    config_override :in_memory_regexp_time_limit, nil
+
+    let!(:owner) { Person.create! }
+
+    before do
+      30.times { owner.posts.create!(title: slow_street) }
+      Mongoid::Config.in_memory_regexp_time_limit = calibrated_limit
+    end
+
+    it 'raises rather than scanning every loaded document' do
+      expect do
+        owner.posts.delete_all(title: { '$regex' => slow_pattern })
+      end.to raise_error(Mongoid::Errors::InMemoryRegexpTimeout)
+    end
+
+    it 'leaves the documents in place' do
+      expect do
+        owner.posts.delete_all(title: { '$regex' => slow_pattern })
+      end.to raise_error(Mongoid::Errors::InMemoryRegexpTimeout)
+
+      expect(Post.count).to eq(30)
+    end
+  end
+end

data/spec/integration/query_operator_guard_spec.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe 'query operator injection guard' do
+  let(:js) { 'this.name == "admin"' }
+
+  let(:nested_function) do
+    { '$expr' => { '$function' => { 'body' => 'function() { return true; }', 'args' => [], 'lang' => 'js' } } }
+  end
+
+  let(:band) { Band.create!(name: 'Depeche Mode') }
+
+  context 'with default configuration' do
+    it 'rejects unsafe operators without any opt-out' do
+      expect(Mongoid.allow_unsafe_query_operators).to be false
+    end
+
+    context 'when querying with where' do
+      it 'rejects a top-level $where' do
+        expect { Band.where('$where' => js).first }.to raise_error(Mongoid::Errors::InvalidQuery)
+      end
+
+      it 'rejects a $function nested in $expr' do
+        expect { Band.where(nested_function).first }.to raise_error(Mongoid::Errors::InvalidQuery)
+      end
+
+      it 'permits an ordinary query' do
+        expect(Band.where(name: band.name).first).to eq(band)
+      end
+    end
+
+    context 'when querying with find_by' do
+      it 'rejects a top-level $where' do
+        expect { Band.find_by('$where' => js) }.to raise_error(Mongoid::Errors::InvalidQuery)
+      end
+
+      it 'rejects a $function nested in $expr' do
+        expect { Band.find_by(nested_function) }.to raise_error(Mongoid::Errors::InvalidQuery)
+      end
+    end
+
+    context 'when querying with find_or_create_by' do
+      it 'rejects a top-level $where' do
+        expect { Band.find_or_create_by('$where' => js) }.to raise_error(Mongoid::Errors::InvalidQuery)
+      end
+    end
+
+    context 'when querying with find_or_initialize_by' do
+      it 'rejects a top-level $where' do
+        expect { Band.find_or_initialize_by('$where' => js) }.to raise_error(Mongoid::Errors::InvalidQuery)
+      end
+    end
+
+    context 'when querying an association with find_or_create_by' do
+      it 'rejects a top-level $where' do
+        expect do
+          band.records.find_or_create_by('$where' => js)
+        end.to raise_error(Mongoid::Errors::InvalidQuery)
+      end
+    end
+
+    context 'when querying with a logical operator' do
+      it 'rejects a $where smuggled through or' do
+        expect { Band.or('$where' => js).first }.to raise_error(Mongoid::Errors::InvalidQuery)
+      end
+
+      it 'rejects a $where smuggled through any_of' do
+        expect { Band.any_of('$where' => js).first }.to raise_error(Mongoid::Errors::InvalidQuery)
+      end
+    end
+  end
+
+  context 'when allow_unsafe_query_operators is true' do
+    config_override :allow_unsafe_query_operators, true
+
+    it 'permits a top-level $where' do
+      expect(Band.where('$where' => "this.name == '#{band.name}'").first).to eq(band)
+    end
+
+    it 'permits a $function nested in $expr' do
+      expect { Band.where(nested_function).first }.not_to raise_error
+    end
+
+    it 'permits a $where smuggled through or' do
+      expect { Band.or('$where' => js).first }.not_to raise_error
+    end
+  end
+end

data/spec/mongoid/association/referenced/has_and_belongs_to_many/proxy_spec.rb

@@ -2268,6 +2268,13 @@ describe Mongoid::Association::Referenced::HasAndBelongsToMany::Proxy do
     end
   end
 
+  # Whether the conditions carry a pattern decides which branch of remove_all
+  # runs, so anything that has to behave the same on both is exercised twice.
+  conditions_variants = [
+    [ 'no regular expression', nil ],
+    [ 'a regular expression', { name: /Test/ } ]
+  ].freeze
+
   %i[ delete_all destroy_all ].each do |method|
     describe "\##{method}" do
       context 'when the relation is not polymorphic' do
@@ -2339,6 +2346,47 @@ describe Mongoid::Association::Referenced::HasAndBelongsToMany::Proxy do
             expect(person.preferences).to eq([])
           end
         end
+
+        context 'when the base holds the foreign keys' do
+          # unbind_one pulls ids out of the base's foreign key array and marks
+          # the base dirty, so which documents get unbound is visible here.
+          # Only what the association already held is unbound, and that has to
+          # hold whether or not the conditions carry a regular expression:
+          # scanning under a regexp budget loads the association first, and
+          # that load must not turn into an unbind of its own. See the note in
+          # HasMany::Proxy#remove_all_bounded.
+          let!(:person) do
+            Person.create!.tap do |base|
+              base.preferences.create!(name: 'Testing')
+              base.preferences.create!(name: 'Test')
+            end
+          end
+
+          let(:ids) { person.preference_ids.dup }
+
+          conditions_variants.each do |description, conditions|
+            context "when the conditions carry #{description}" do
+              context 'when the association is already in memory' do
+                it 'unbinds the deleted documents' do
+                  person.preferences.send(method, conditions)
+                  expect(person.preference_ids).to eq([])
+                  expect(person).to be_changed
+                end
+              end
+
+              context 'when the association has not been loaded' do
+                let(:cold) { Person.find(person._id) }
+
+                it 'leaves the foreign keys alone' do
+                  expect(ids.length).to eq(2)
+                  cold.preferences.send(method, conditions)
+                  expect(cold.preference_ids).to eq(ids)
+                  expect(cold).not_to be_changed
+                end
+              end
+            end
+          end
+        end
       end
     end
   end

data/spec/mongoid/association/referenced/has_many/proxy_spec.rb

@@ -2281,6 +2281,151 @@ describe Mongoid::Association::Referenced::HasMany::Proxy do
             expect(person.posts).to eq([])
           end
         end
+
+        context 'when a document is appended with an id that is already loaded' do
+          # Iterating the association moves an appended document out of _added
+          # without adding it to _loaded, so the scan for matches and the
+          # removal that follows see different instances for the same id.
+          let(:person) { Person.create! }
+          let!(:post) { person.posts.create!(title: 'Testing') }
+          let(:reloaded) { Person.find(person._id) }
+
+          before do
+            reloaded.posts.to_a
+            reloaded.posts._target.push(
+              Post.new(_id: post._id, title: 'Testing', person_id: reloaded._id)
+            )
+          end
+
+          shared_examples 'removes the document once' do
+            it 'sets the association locally' do
+              reloaded.posts.send(method, conditions)
+              expect(reloaded.posts).to eq([])
+            end
+
+            it 'deletes the documents from the database' do
+              reloaded.posts.send(method, conditions)
+              expect(Post.count).to eq(0)
+            end
+          end
+
+          context 'when the conditions carry no regular expression' do
+            let(:conditions) { { title: 'Testing' } }
+
+            it_behaves_like 'removes the document once'
+          end
+
+          context 'when the conditions carry a regular expression' do
+            let(:conditions) { { title: /Testing/ } }
+
+            it_behaves_like 'removes the document once'
+          end
+        end
+
+        context 'when an appended document disagrees with the loaded one' do
+          # Only the appended instance matches, so the removal has to drop that
+          # id from both _loaded and _added. Removing by id and stopping at the
+          # first hash that has it would take the loaded instance instead and
+          # leave the unbound one behind.
+          let(:person) { Person.create! }
+          let!(:post) { person.posts.create!(title: 'Other') }
+          let(:reloaded) { Person.find(person._id) }
+
+          before do
+            reloaded.posts.to_a
+            reloaded.posts._target.push(
+              Post.new(_id: post._id, title: 'Testing', person_id: reloaded._id)
+            )
+          end
+
+          it 'deletes nothing from the database' do
+            expect(reloaded.posts.send(method, { title: /Testing/ })).to eq(0)
+            expect(Post.where(title: 'Other').count).to eq(1)
+          end
+
+          it 'leaves no unbound document in the association' do
+            reloaded.posts.send(method, { title: /Testing/ })
+            expect(reloaded.posts._target.in_memory.map(&:person_id)).not_to include(nil)
+          end
+        end
+
+        context 'when a nested selector is evaluated while the association loads' do
+          # Loading the association runs find callbacks, and application code in
+          # one of those can evaluate a selector of its own. A scope opened
+          # around the load would mark it as having nothing to bound, and that
+          # suppresses the decision for the nested selector too, leaving any
+          # pattern in it unguarded.
+          config_override :in_memory_regexp_time_limit, 5.0
+
+          let(:person) { Person.create! }
+          let(:reloaded) { Person.find(person._id) }
+
+          before do
+            person.posts.create!(title: 'Testing')
+            person.posts.create!(title: 'Keep')
+            reloaded
+          end
+
+          it 'leaves the nested scope free to establish its own budget' do
+            seen = []
+            allow(Mongoid::Factory).to receive(:from_db).and_wrap_original do |original, *args|
+              Mongoid::Matcher::RegexpBudget.open('title' => /Testing/) do
+                seen << Mongoid::Matcher::RegexpBudget.remaining
+              end
+              original.call(*args)
+            end
+
+            reloaded.posts.send(method, { title: 'Testing' })
+
+            expect(seen).not_to be_empty
+            expect(seen).not_to include(nil)
+          end
+        end
+
+        if method == :delete_all
+          context 'when the association has not been loaded' do
+            let(:person) { Person.create! }
+            let(:reloaded) { Person.find(person._id) }
+            let(:subscriber) { Mrss::EventSubscriber.new }
+
+            before do
+              person.posts.create!(title: 'Testing')
+              person.posts.create!(title: 'Test')
+              reloaded
+            end
+
+            def commands_for(conditions)
+              Person.collection.client.subscribe(Mongo::Monitoring::COMMAND, subscriber)
+              reloaded.posts.delete_all(conditions)
+              subscriber.started_events.map(&:command_name)
+            ensure
+              Person.collection.client.unsubscribe(Mongo::Monitoring::COMMAND, subscriber)
+            end
+
+            context 'when the conditions carry no regular expression' do
+              # Nothing to bound, so the scan for matches stays where it was,
+              # after the delete, where the association it loads comes back
+              # empty. Moving it ahead of the delete would transfer the whole
+              # association for an operation that need send nothing over the
+              # wire.
+              it 'deletes before loading the association' do
+                expect(commands_for(nil)).to eq(%w[ delete find ])
+              end
+            end
+
+            context 'when the conditions carry a regular expression' do
+              # The scan has to run under a budget, and before anything is
+              # deleted, so that a budget which runs out leaves the database
+              # untouched. What it scans is what is already in memory: loading
+              # would trade the matching cost the budget exists to bound for an
+              # unbounded amount of memory, since /.*/ is both cheap to supply
+              # and matches every document in the association.
+              it 'deletes without loading the association' do
+                expect(commands_for({ title: /Test/ })).to eq(%w[ delete ])
+              end
+            end
+          end
+        end
       end
 
       context "when the association is polymorphic" do