frozen_record 0.23.0 → 0.24.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f377e3f70ea220a100dda1e18d3d01cb4a09958a13581283d20df91a1f369a89
-  data.tar.gz: 136d29c3521aea08c5688a59bf56edfcc231707915c9e67fa9d8e05ad59e3266
+  metadata.gz: b99597156ea028c8d0cbd66024b901d55a5e26679030266367caa8ec81353946
+  data.tar.gz: 12e4756a7a49d32010298b4320f447d9b18afd76efe969a1f3a32e7d890592de
 SHA512:
-  metadata.gz: 4eda9868623fa545ca344ffae2e5849b1350458a55f764924acea2891cc4a7dadb36d7344d13928c34e4d0416950f70f9075cf63130e132aea5fbdb0b54fb105
-  data.tar.gz: 481858eac4b1a34f2247aa4173afa145bb535ce07c1e54594287b0749fff0a79e538133fa51eb84b3ffac8e40eaf82af703e7a9e63131938cfe87d4e2fab1e80
+  metadata.gz: fcb9c7aa9c820dfa50b431d36bef98d39488be55fa034edaed7aefde063a919134af0d945e3233ca5a6efbc3c3798245a5ea5741b5ae1f4d8dad1b0189b1bf7e
+  data.tar.gz: df184ce79d274a9fe8a6fcb70c9548bb18f17c1e8da143c88d2ab9f2aedec746314a21ee2e591e191fd248f8d5e332dfdcc379f316d09fcb070a5284724f43ad

data/.github/workflows/main.yml

@@ -6,26 +6,18 @@ jobs:
   build:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
+        ruby: [ '2.5', '2.6', '2.7', '3.0' , '3.1']
         minimal: [ false, true ]
     name: Ruby ${{ matrix.ruby }} tests, minimal=${{ matrix.minimal }}
     steps:
       - uses: actions/checkout@v2
-      - name: Setup Ruby
+      - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
-      - uses: actions/cache@v2
-        with:
-          path: vendor/bundle
-          key: ${{ runner.os }}-${{ matrix.ruby }}-gems-${{ hashFiles('Gemfile', 'frozen_record.gemspec') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.ruby }}-gems-
-      - name: Bundle install
-        run: |
-          gem install bundler
-          bundle install --jobs 4 --retry 3 --path=vendor/bundle
+          bundler-cache: true
       - name: Run tests
         env:
           MINIMAL: ${{ matrix.minimal }}

data/CHANGELOG.md

@@ -0,0 +1,17 @@
+# Unreleased
+
+# v0.24.1
+
+- Fix index selection not applying some restrictions.
+
+# v0.24.0 (yanked)
+
+- Improve index selection and combinaison. Should significantly help with performance in some cases.
+- Implement `max_records_scan` to reject slow queries.
+- Only load `Railtie` integration if `Rails::Railtie` is defined
+- Allow granular fixture unloading
+- Fix a bug affecting older bootsnap versions
+
+# v0.23.0
+
+NO DATA

data/README.md

@@ -3,7 +3,7 @@
 [![Build Status](https://secure.travis-ci.org/byroot/frozen_record.svg)](http://travis-ci.org/byroot/frozen_record)
 [![Gem Version](https://badge.fury.io/rb/frozen_record.svg)](http://badge.fury.io/rb/frozen_record)
 
-ActiveRecord-like interface for **read only** access to static data files.
+Activec Record-like interface for **read only** access to static data files of reasonable size.
 
 ## Installation
 
@@ -21,7 +21,7 @@ Or install it yourself as:
 
 ## Models definition
 
-Just like with ActiveRecord, your models need to inherits from `FrozenRecord::Base`:
+Just like with Active Record, your models need to inherits from `FrozenRecord::Base`:
 
 ```ruby
 class Country < FrozenRecord::Base
@@ -72,7 +72,7 @@ end
 
 ## Query interface
 
-FrozenRecord aim to replicate only modern ActiveRecord querying interface, and only the non "string typed" ones.
+FrozenRecord aim to replicate only modern Active Record querying interface, and only the non "string typed" ones.
 
 e.g
 ```ruby
@@ -154,6 +154,13 @@ Composite index keys are not supported.
 
 The primary key isn't indexed by default.
 
+## Limitations
+
+Frozen Record is not meant to operate or large unindexed datasets.
+
+To ensure that it doesn't happen by accident, you can set `FrozenRecord::Base.max_records_scan = 500` (or whatever limit makes sense to you), in your development and test environments.
+This setting will cause Frozen Record to raise an error if it has to scan more than `max_records_scan` records. This property can also be set on a per model basis.
+
 ## Configuration
 
 ### Reloading

data/lib/frozen_record/backends/empty.yml

@@ -0,0 +1 @@
+true

data/lib/frozen_record/backends/yaml.rb

@@ -49,7 +49,7 @@ module FrozenRecord
       private
 
       supports_freeze = begin
-        YAML.load_file(File.expand_path('../empty.json', __FILE__), freeze: true)
+        YAML.load_file(File.expand_path('../empty.yml', __FILE__), freeze: true)
       rescue ArgumentError
         false
       end

data/lib/frozen_record/base.rb

@@ -4,6 +4,8 @@ require 'active_support/descendants_tracker'
 require 'frozen_record/backends'
 
 module FrozenRecord
+  SlowQuery = Class.new(StandardError)
+
   class Base
     extend ActiveSupport::DescendantsTracker
     extend ActiveModel::Naming
@@ -15,6 +17,7 @@ module FrozenRecord
 
     class_attribute :base_path, :primary_key, :backend, :auto_reloading, :default_attributes, instance_accessor: false
     class_attribute :index_definitions, instance_accessor: false
+    class_attribute :max_records_scan, instance_accessor: false
     self.index_definitions = {}.freeze
 
     self.primary_key = 'id'

data/lib/frozen_record/scope.rb

@@ -173,6 +173,8 @@ module FrozenRecord
       sort_records(select_records(@klass.load_records))
     end
 
+    ARRAY_INTERSECTION = Array.method_defined?(:intersection)
+
     def select_records(records)
       return records if @where_values.empty? && @where_not_values.empty?
 
@@ -180,13 +182,30 @@ module FrozenRecord
       indexed_where_values, unindexed_where_values = @where_values.partition { |criteria| indices.key?(criteria.first) }
 
       unless indexed_where_values.empty?
-        attribute, value = indexed_where_values.shift
-        records = indices[attribute].query(value)
-        indexed_where_values.each do |(attribute, value)|
-          records &= indices[attribute].query(value)
+        usable_indexes = indexed_where_values.map { |(attribute, value)| [attribute, value, indices[attribute].query(value)] }
+        usable_indexes.sort_by! { |r| r[2].size }
+        records = usable_indexes.shift.last
+
+        # If the index is 5 times bigger that the current set of records it's not worth doing an array intersection.
+        # The value is somewhat arbitrary and could be adjusted.
+        useless_indexes, usable_indexes = usable_indexes.partition { |_, _, indexed_records| indexed_records.size > records.size * 5 }
+        unindexed_where_values += useless_indexes.map { |a| a.first(2) }
+
+        unless usable_indexes.empty?
+          if ARRAY_INTERSECTION
+            records = records.intersection(*usable_indexes.map(&:last))
+          else
+            usable_indexes.each do |_, _, indexed_records|
+              records &= indexed_records
+            end
+          end
         end
       end
 
+      if @klass.max_records_scan && records.size > @klass.max_records_scan
+        raise SlowQuery, "Scanning #{records.size} records is too slow, the allowed maximum is #{@klass.max_records_scan}. Try to find a better index or consider an alternative storage"
+      end
+
       records.select do |record|
         unindexed_where_values.all? { |attr, matcher| matcher.match?(record[attr]) } &&
         !@where_not_values.any? { |attr, matcher| matcher.match?(record[attr]) }

data/lib/frozen_record/test_helper.rb

@@ -8,9 +8,7 @@ module FrozenRecord
       def load_fixture(model_class, alternate_base_path)
         @cache ||= {}
 
-        unless model_class < FrozenRecord::Base
-          raise ArgumentError, "Model class (#{model_class}) does not inherit from #{FrozenRecord::Base}"
-        end
+        ensure_model_class_is_frozenrecord(model_class)
 
         return if @cache.key?(model_class)
 
@@ -20,20 +18,32 @@ module FrozenRecord
         model_class.load_records(force: true)
       end
 
-      def unload_fixtures
+      def unload_fixture(model_class)
         return unless defined?(@cache) && @cache
 
-        @cache.each do |model_class, old_base_path|
-          model_class.base_path = old_base_path
-          model_class.load_records(force: true)
-        end
+        ensure_model_class_is_frozenrecord(model_class)
+
+        return unless @cache.key?(model_class)
+
+        old_base_path = @cache[model_class]
+        model_class.base_path = old_base_path
+        model_class.load_records(force: true)
 
-        @cache = nil
+        @cache.delete(model_class)
+      end
+
+      def unload_fixtures
+        return unless defined?(@cache) && @cache
+
+        @cache.keys.each { |model_class| unload_fixture(model_class) }
       end
 
       private
 
       def ensure_model_class_is_frozenrecord(model_class)
+        unless model_class < FrozenRecord::Base
+          raise ArgumentError, "Model class (#{model_class}) does not inherit from #{FrozenRecord::Base}"
+        end
       end
     end
   end

data/lib/frozen_record/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FrozenRecord
-  VERSION = '0.23.0'
+  VERSION = '0.24.1'
 end

data/lib/frozen_record.rb

@@ -2,4 +2,4 @@
 
 require 'frozen_record/minimal'
 require 'frozen_record/serialization'
-require 'frozen_record/railtie' if defined?(Rails)
+require 'frozen_record/railtie' if defined?(Rails::Railtie)

data/spec/fixtures/continents.yml.erb

@@ -0,0 +1,9 @@
+---
+- id: 1
+  name: Africa
+
+- id: 2
+  name: Antarctica
+
+- id: 3
+  name: Asia

data/spec/fixtures/test_helper/continents.yml.erb

@@ -0,0 +1,3 @@
+---
+- id: 1
+  name: Some continent

data/spec/scope_spec.rb

@@ -474,4 +474,19 @@ describe 'querying' do
 
   end
 
+  context 'when max_records_scan is set' do
+
+    around :each do |example|
+      FrozenRecord::Base.max_records_scan = 1
+      example.run
+      FrozenRecord::Base.max_records_scan = nil
+    end
+
+    it 'raises on slow queries' do
+      expect {
+        Country.where(king: "Louis").to_a
+      }.to raise_error(FrozenRecord::SlowQuery)
+    end
+
+  end
 end

data/spec/support/continent.rb

@@ -0,0 +1,2 @@
+class Continent < FrozenRecord::Base
+end

data/spec/test_helper_spec.rb

@@ -38,19 +38,35 @@ describe 'test fixture loading' do
     end
   end
 
+  describe '.unload_fixture' do
+    it 'restores the default fixtures for the specified model class' do
+      test_fixtures_base_path = File.join(File.dirname(__FILE__), 'fixtures', 'test_helper')
+
+      FrozenRecord::TestHelper.load_fixture(Continent, test_fixtures_base_path)
+      FrozenRecord::TestHelper.load_fixture(Country, test_fixtures_base_path)
+      FrozenRecord::TestHelper.unload_fixture(Country)
+
+      expect(Continent.count).to be == 1
+      expect(Country.count).to be == 3
+    end
+  end
+
   describe '.unload_fixtures' do
     it 'restores the default fixtures' do
       test_fixtures_base_path = File.join(File.dirname(__FILE__), 'fixtures', 'test_helper')
 
+      FrozenRecord::TestHelper.load_fixture(Continent, test_fixtures_base_path)
       FrozenRecord::TestHelper.load_fixture(Country, test_fixtures_base_path)
       FrozenRecord::TestHelper.unload_fixtures
 
+      expect(Continent.count).to be == 3
       expect(Country.count).to be == 3
     end
 
     it 'does has no effect if no alternate fixtures were loaded' do
       FrozenRecord::TestHelper.unload_fixtures
 
+      expect(Continent.count).to be == 3
       expect(Country.count).to be == 3
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: frozen_record
 version: !ruby/object:Gem::Version
-  version: 0.23.0
+  version: 0.24.1
 platform: ruby
 authors:
 - Jean Boussier
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-10-22 00:00:00.000000000 Z
+date: 2022-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -76,6 +76,7 @@ files:
 - ".github/workflows/main.yml"
 - ".gitignore"
 - ".rspec"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -88,6 +89,7 @@ files:
 - lib/frozen_record.rb
 - lib/frozen_record/backends.rb
 - lib/frozen_record/backends/empty.json
+- lib/frozen_record/backends/empty.yml
 - lib/frozen_record/backends/json.rb
 - lib/frozen_record/backends/yaml.rb
 - lib/frozen_record/base.rb
@@ -101,7 +103,9 @@ files:
 - lib/frozen_record/version.rb
 - spec/fixtures/animals.json
 - spec/fixtures/cars.yml
+- spec/fixtures/continents.yml.erb
 - spec/fixtures/countries.yml.erb
+- spec/fixtures/test_helper/continents.yml.erb
 - spec/fixtures/test_helper/countries.yml.erb
 - spec/frozen_record_spec.rb
 - spec/scope_spec.rb
@@ -109,6 +113,7 @@ files:
 - spec/support/abstract_model.rb
 - spec/support/animal.rb
 - spec/support/car.rb
+- spec/support/continent.rb
 - spec/support/country.rb
 - spec/test_helper_spec.rb
 homepage: https://github.com/byroot/frozen_record
@@ -130,14 +135,16 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: ActiveRecord like interface to read only access and query static YAML files
 test_files:
 - spec/fixtures/animals.json
 - spec/fixtures/cars.yml
+- spec/fixtures/continents.yml.erb
 - spec/fixtures/countries.yml.erb
+- spec/fixtures/test_helper/continents.yml.erb
 - spec/fixtures/test_helper/countries.yml.erb
 - spec/frozen_record_spec.rb
 - spec/scope_spec.rb
@@ -145,5 +152,6 @@ test_files:
 - spec/support/abstract_model.rb
 - spec/support/animal.rb
 - spec/support/car.rb
+- spec/support/continent.rb
 - spec/support/country.rb
 - spec/test_helper_spec.rb