frozen_record 0.23.0 → 0.25.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f377e3f70ea220a100dda1e18d3d01cb4a09958a13581283d20df91a1f369a89
-  data.tar.gz: 136d29c3521aea08c5688a59bf56edfcc231707915c9e67fa9d8e05ad59e3266
+  metadata.gz: 2c4f3cb4248d83050752933b1a2c48d0bf0914a73e695932650a03cae9feffa0
+  data.tar.gz: fb8517287957255269ffec7ab7bac5740c6d7e45b4f1c3c0b321ce4fc5badf4f
 SHA512:
-  metadata.gz: 4eda9868623fa545ca344ffae2e5849b1350458a55f764924acea2891cc4a7dadb36d7344d13928c34e4d0416950f70f9075cf63130e132aea5fbdb0b54fb105
-  data.tar.gz: 481858eac4b1a34f2247aa4173afa145bb535ce07c1e54594287b0749fff0a79e538133fa51eb84b3ffac8e40eaf82af703e7a9e63131938cfe87d4e2fab1e80
+  metadata.gz: 1d66248c13f74ccaa779db89359467e54c1ee18979683fcd42e62485ac5657345115985329f07be02f0ffe6f347334646e26bb920df1b1efad2d3ef55ab24d74
+  data.tar.gz: f395e256aca6a133fe684f3e28a94cbdca4f70ea6a2c565a63f30bb4f52144f62529a6b20c24d806e97a4ba0bb11a5ca0e2402d521c700f13bc5b1bb49331578

data/.github/workflows/main.yml

@@ -6,26 +6,18 @@ jobs:
   build:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
+        ruby: [ '2.5', '2.6', '2.7', '3.0' , '3.1']
         minimal: [ false, true ]
     name: Ruby ${{ matrix.ruby }} tests, minimal=${{ matrix.minimal }}
     steps:
       - uses: actions/checkout@v2
-      - name: Setup Ruby
+      - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
-      - uses: actions/cache@v2
-        with:
-          path: vendor/bundle
-          key: ${{ runner.os }}-${{ matrix.ruby }}-gems-${{ hashFiles('Gemfile', 'frozen_record.gemspec') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.ruby }}-gems-
-      - name: Bundle install
-        run: |
-          gem install bundler
-          bundle install --jobs 4 --retry 3 --path=vendor/bundle
+          bundler-cache: true
       - name: Run tests
         env:
           MINIMAL: ${{ matrix.minimal }}

data/CHANGELOG.md

@@ -0,0 +1,26 @@
+# Unreleased
+
+# v0.25.1
+
+- More reliable way to disable max_records_scan checks when loading records.
+
+# v0.25.0
+
+- Disable max_records_scan checks when loading records.
+- Add `FrozenRecord::Base.with_max_record_scan` for more easily allowing larger amount in specific tests.
+
+# v0.24.1
+
+- Fix index selection not applying some restrictions.
+
+# v0.24.0 (yanked)
+
+- Improve index selection and combinaison. Should significantly help with performance in some cases.
+- Implement `max_records_scan` to reject slow queries.
+- Only load `Railtie` integration if `Rails::Railtie` is defined
+- Allow granular fixture unloading
+- Fix a bug affecting older bootsnap versions
+
+# v0.23.0
+
+NO DATA

data/README.md

@@ -3,7 +3,7 @@
 [![Build Status](https://secure.travis-ci.org/byroot/frozen_record.svg)](http://travis-ci.org/byroot/frozen_record)
 [![Gem Version](https://badge.fury.io/rb/frozen_record.svg)](http://badge.fury.io/rb/frozen_record)
 
-ActiveRecord-like interface for **read only** access to static data files.
+Activec Record-like interface for **read only** access to static data files of reasonable size.
 
 ## Installation
 
@@ -21,7 +21,7 @@ Or install it yourself as:
 
 ## Models definition
 
-Just like with ActiveRecord, your models need to inherits from `FrozenRecord::Base`:
+Just like with Active Record, your models need to inherits from `FrozenRecord::Base`:
 
 ```ruby
 class Country < FrozenRecord::Base
@@ -72,7 +72,7 @@ end
 
 ## Query interface
 
-FrozenRecord aim to replicate only modern ActiveRecord querying interface, and only the non "string typed" ones.
+FrozenRecord aim to replicate only modern Active Record querying interface, and only the non "string typed" ones.
 
 e.g
 ```ruby
@@ -154,6 +154,13 @@ Composite index keys are not supported.
 
 The primary key isn't indexed by default.
 
+## Limitations
+
+Frozen Record is not meant to operate or large unindexed datasets.
+
+To ensure that it doesn't happen by accident, you can set `FrozenRecord::Base.max_records_scan = 500` (or whatever limit makes sense to you), in your development and test environments.
+This setting will cause Frozen Record to raise an error if it has to scan more than `max_records_scan` records. This property can also be set on a per model basis.
+
 ## Configuration
 
 ### Reloading

data/lib/frozen_record/backends/empty.yml

@@ -0,0 +1 @@
+true

data/lib/frozen_record/backends/yaml.rb

@@ -49,7 +49,7 @@ module FrozenRecord
       private
 
       supports_freeze = begin
-        YAML.load_file(File.expand_path('../empty.json', __FILE__), freeze: true)
+        YAML.load_file(File.expand_path('../empty.yml', __FILE__), freeze: true)
       rescue ArgumentError
         false
       end

data/lib/frozen_record/base.rb

@@ -4,6 +4,20 @@ require 'active_support/descendants_tracker'
 require 'frozen_record/backends'
 
 module FrozenRecord
+  SlowQuery = Class.new(StandardError)
+
+  class << self
+    attr_accessor :enforce_max_records_scan
+
+    def ignore_max_records_scan
+      previous = enforce_max_records_scan
+      yield
+    ensure
+      self.enforce_max_records_scan = previous
+    end
+  end
+  @enforce_max_records_scan = true
+
   class Base
     extend ActiveSupport::DescendantsTracker
     extend ActiveModel::Naming
@@ -15,6 +29,7 @@ module FrozenRecord
 
     class_attribute :base_path, :primary_key, :backend, :auto_reloading, :default_attributes, instance_accessor: false
     class_attribute :index_definitions, instance_accessor: false
+    class_attribute :max_records_scan, instance_accessor: false
     self.index_definitions = {}.freeze
 
     self.primary_key = 'id'
@@ -42,6 +57,14 @@ module FrozenRecord
     end
 
     class << self
+      def with_max_records_scan(value)
+        previous_max_records_scan = max_records_scan
+        self.max_records_scan = value
+        yield
+      ensure
+        self.max_records_scan = previous_max_records_scan
+      end
+
       alias_method :set_default_attributes, :default_attributes=
       private :set_default_attributes
       def default_attributes=(default_attributes)
@@ -145,7 +168,7 @@ module FrozenRecord
           end
           @attributes = list_attributes(records).freeze
           define_attribute_methods(@attributes.to_a)
-          records = records.map { |r| load(r) }.freeze
+          records = FrozenRecord.ignore_max_records_scan { records.map { |r| load(r) }.freeze }
           index_definitions.values.each { |index| index.build(records) }
           records
         end

data/lib/frozen_record/scope.rb

@@ -173,6 +173,8 @@ module FrozenRecord
       sort_records(select_records(@klass.load_records))
     end
 
+    ARRAY_INTERSECTION = Array.method_defined?(:intersection)
+
     def select_records(records)
       return records if @where_values.empty? && @where_not_values.empty?
 
@@ -180,13 +182,30 @@ module FrozenRecord
       indexed_where_values, unindexed_where_values = @where_values.partition { |criteria| indices.key?(criteria.first) }
 
       unless indexed_where_values.empty?
-        attribute, value = indexed_where_values.shift
-        records = indices[attribute].query(value)
-        indexed_where_values.each do |(attribute, value)|
-          records &= indices[attribute].query(value)
+        usable_indexes = indexed_where_values.map { |(attribute, value)| [attribute, value, indices[attribute].query(value)] }
+        usable_indexes.sort_by! { |r| r[2].size }
+        records = usable_indexes.shift.last
+
+        # If the index is 5 times bigger that the current set of records it's not worth doing an array intersection.
+        # The value is somewhat arbitrary and could be adjusted.
+        useless_indexes, usable_indexes = usable_indexes.partition { |_, _, indexed_records| indexed_records.size > records.size * 5 }
+        unindexed_where_values += useless_indexes.map { |a| a.first(2) }
+
+        unless usable_indexes.empty?
+          if ARRAY_INTERSECTION
+            records = records.intersection(*usable_indexes.map(&:last))
+          else
+            usable_indexes.each do |_, _, indexed_records|
+              records &= indexed_records
+            end
+          end
         end
       end
 
+      if FrozenRecord.enforce_max_records_scan && @klass.max_records_scan && records.size > @klass.max_records_scan
+        raise SlowQuery, "Scanning #{records.size} records is too slow, the allowed maximum is #{@klass.max_records_scan}. Try to find a better index or consider an alternative storage"
+      end
+
       records.select do |record|
         unindexed_where_values.all? { |attr, matcher| matcher.match?(record[attr]) } &&
         !@where_not_values.any? { |attr, matcher| matcher.match?(record[attr]) }

data/lib/frozen_record/test_helper.rb

@@ -8,9 +8,7 @@ module FrozenRecord
       def load_fixture(model_class, alternate_base_path)
         @cache ||= {}
 
-        unless model_class < FrozenRecord::Base
-          raise ArgumentError, "Model class (#{model_class}) does not inherit from #{FrozenRecord::Base}"
-        end
+        ensure_model_class_is_frozenrecord(model_class)
 
         return if @cache.key?(model_class)
 
@@ -20,20 +18,32 @@ module FrozenRecord
         model_class.load_records(force: true)
       end
 
-      def unload_fixtures
+      def unload_fixture(model_class)
         return unless defined?(@cache) && @cache
 
-        @cache.each do |model_class, old_base_path|
-          model_class.base_path = old_base_path
-          model_class.load_records(force: true)
-        end
+        ensure_model_class_is_frozenrecord(model_class)
+
+        return unless @cache.key?(model_class)
+
+        old_base_path = @cache[model_class]
+        model_class.base_path = old_base_path
+        model_class.load_records(force: true)
 
-        @cache = nil
+        @cache.delete(model_class)
+      end
+
+      def unload_fixtures
+        return unless defined?(@cache) && @cache
+
+        @cache.keys.each { |model_class| unload_fixture(model_class) }
       end
 
       private
 
       def ensure_model_class_is_frozenrecord(model_class)
+        unless model_class < FrozenRecord::Base
+          raise ArgumentError, "Model class (#{model_class}) does not inherit from #{FrozenRecord::Base}"
+        end
       end
     end
   end

data/lib/frozen_record/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FrozenRecord
-  VERSION = '0.23.0'
+  VERSION = '0.25.1'
 end

data/lib/frozen_record.rb

@@ -2,4 +2,4 @@
 
 require 'frozen_record/minimal'
 require 'frozen_record/serialization'
-require 'frozen_record/railtie' if defined?(Rails)
+require 'frozen_record/railtie' if defined?(Rails::Railtie)

data/spec/fixtures/continents.yml.erb

@@ -0,0 +1,9 @@
+---
+- id: 1
+  name: Africa
+
+- id: 2
+  name: Antarctica
+
+- id: 3
+  name: Asia

data/spec/fixtures/prices.yml.erb

@@ -0,0 +1,13 @@
+<% 100.times.map { |i| "plan_#{i}"}.each do |plan_name| %>
+  <% ["base", "online", "retail"].each do |type| %>
+    <% [nil, "USD", "EUR", "GBP", "JPY"].each do |currency| %>
+     <% ["monthly", "annual", "biennial", "triennial"].each do |period| %>
+- plan_name: <%= plan_name %>
+  type: <%= type %>
+  period: <%= period %>
+  currency: <%= currency %>
+  amount: 10.0
+      <% end %>
+    <% end %>
+  <% end %>
+<% end %>

data/spec/fixtures/test_helper/continents.yml.erb

@@ -0,0 +1,3 @@
+---
+- id: 1
+  name: Some continent

data/spec/scope_spec.rb

@@ -474,4 +474,30 @@ describe 'querying' do
 
   end
 
+  context 'when max_records_scan is set' do
+
+    it 'raises on slow queries' do
+      expect {
+        FrozenRecord::Base.with_max_records_scan(1) do
+          Country.where(king: "Louis").to_a
+        end
+      }.to raise_error(FrozenRecord::SlowQuery)
+    end
+
+    it 'is accurate' do
+      FrozenRecord::Base.with_max_records_scan(60) do
+        expect(Price.count).to be == 6_000
+
+        Price.where(plan_name: "plan_24", currency: [nil, "EUR"], period: "monthly", type: "base").each do |price|
+          expect(price.plan_name).to be == "plan_24"
+          unless price.currency.nil?
+            expect(price.currency).to be  == "EUR"
+          end
+          expect(price.period).to be == "monthly"
+          expect(price.type).to be == "base"
+        end
+      end
+    end
+
+  end
 end

data/spec/support/continent.rb

@@ -0,0 +1,2 @@
+class Continent < FrozenRecord::Base
+end

data/spec/support/price.rb

@@ -0,0 +1,13 @@
+class Price < FrozenRecord::Base
+  add_index :plan_name
+  add_index :currency
+end
+
+module Compact
+  class Price < ::Price
+    include FrozenRecord::Compact
+    def self.file_path
+      superclass.file_path
+    end
+  end
+end

data/spec/test_helper_spec.rb

@@ -38,19 +38,35 @@ describe 'test fixture loading' do
     end
   end
 
+  describe '.unload_fixture' do
+    it 'restores the default fixtures for the specified model class' do
+      test_fixtures_base_path = File.join(File.dirname(__FILE__), 'fixtures', 'test_helper')
+
+      FrozenRecord::TestHelper.load_fixture(Continent, test_fixtures_base_path)
+      FrozenRecord::TestHelper.load_fixture(Country, test_fixtures_base_path)
+      FrozenRecord::TestHelper.unload_fixture(Country)
+
+      expect(Continent.count).to be == 1
+      expect(Country.count).to be == 3
+    end
+  end
+
   describe '.unload_fixtures' do
     it 'restores the default fixtures' do
       test_fixtures_base_path = File.join(File.dirname(__FILE__), 'fixtures', 'test_helper')
 
+      FrozenRecord::TestHelper.load_fixture(Continent, test_fixtures_base_path)
       FrozenRecord::TestHelper.load_fixture(Country, test_fixtures_base_path)
       FrozenRecord::TestHelper.unload_fixtures
 
+      expect(Continent.count).to be == 3
       expect(Country.count).to be == 3
     end
 
     it 'does has no effect if no alternate fixtures were loaded' do
       FrozenRecord::TestHelper.unload_fixtures
 
+      expect(Continent.count).to be == 3
       expect(Country.count).to be == 3
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: frozen_record
 version: !ruby/object:Gem::Version
-  version: 0.23.0
+  version: 0.25.1
 platform: ruby
 authors:
 - Jean Boussier
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-10-22 00:00:00.000000000 Z
+date: 2022-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -76,6 +76,7 @@ files:
 - ".github/workflows/main.yml"
 - ".gitignore"
 - ".rspec"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -88,6 +89,7 @@ files:
 - lib/frozen_record.rb
 - lib/frozen_record/backends.rb
 - lib/frozen_record/backends/empty.json
+- lib/frozen_record/backends/empty.yml
 - lib/frozen_record/backends/json.rb
 - lib/frozen_record/backends/yaml.rb
 - lib/frozen_record/base.rb
@@ -101,7 +103,10 @@ files:
 - lib/frozen_record/version.rb
 - spec/fixtures/animals.json
 - spec/fixtures/cars.yml
+- spec/fixtures/continents.yml.erb
 - spec/fixtures/countries.yml.erb
+- spec/fixtures/prices.yml.erb
+- spec/fixtures/test_helper/continents.yml.erb
 - spec/fixtures/test_helper/countries.yml.erb
 - spec/frozen_record_spec.rb
 - spec/scope_spec.rb
@@ -109,7 +114,9 @@ files:
 - spec/support/abstract_model.rb
 - spec/support/animal.rb
 - spec/support/car.rb
+- spec/support/continent.rb
 - spec/support/country.rb
+- spec/support/price.rb
 - spec/test_helper_spec.rb
 homepage: https://github.com/byroot/frozen_record
 licenses:
@@ -130,14 +137,17 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: ActiveRecord like interface to read only access and query static YAML files
 test_files:
 - spec/fixtures/animals.json
 - spec/fixtures/cars.yml
+- spec/fixtures/continents.yml.erb
 - spec/fixtures/countries.yml.erb
+- spec/fixtures/prices.yml.erb
+- spec/fixtures/test_helper/continents.yml.erb
 - spec/fixtures/test_helper/countries.yml.erb
 - spec/frozen_record_spec.rb
 - spec/scope_spec.rb
@@ -145,5 +155,7 @@ test_files:
 - spec/support/abstract_model.rb
 - spec/support/animal.rb
 - spec/support/car.rb
+- spec/support/continent.rb
 - spec/support/country.rb
+- spec/support/price.rb
 - spec/test_helper_spec.rb