frozen_record 0.22.2 → 0.25.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 46d80cfe87901ec547852256bff7eed06ac990a7bb9705b93fa13464354c5af5
-  data.tar.gz: 63f48c770a8e86d867382f0333f1c5d6d48b64ece3b849717fd840945f0781ff
+  metadata.gz: d2be87eaecead5b4f1d4c5e941d48fcd9be163c56f2c9b988ec4d6ff7ea84757
+  data.tar.gz: 0a4a51c609251f3a46ba49e16f85b1d27ce5e52581fb70bd605c8585b1f62fa8
 SHA512:
-  metadata.gz: 0a5c7c217742022ff5d23532cc54c04f040ae83e9e05c191783a49db5012c2c95f2815a85a8743e00bd467bbff9fb451d94ba5a75cdd80bde71dee66ccb2a256
-  data.tar.gz: 587dc2bcd6cec261cd53c9be8cfe696b7f4944d1c55b5cea55945b7826112eed2829a323c617c906dfcd039d72578e4b4ba5cb790a1ba1131277c6076dafcafc
+  metadata.gz: 4326747e566bdb03b700dd5bb8164b550f8d339991754cebeb2a5414f8858a4bb19c074ac2231d4e978be881ed0ff87611c107c69bbc2a21ba498b7d3eb563eb
+  data.tar.gz: f73715f137336bada8a8aad2a3be9d6b4cf87f4de9dae6c2b77463201f7a3e510ae6c7491d9957db9c7d2b6df8280576e7e896eb8bf28b661bff02d32d4634f5

data/.github/workflows/main.yml

@@ -6,26 +6,18 @@ jobs:
   build:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
+        ruby: [ '2.5', '2.6', '2.7', '3.0' , '3.1']
         minimal: [ false, true ]
     name: Ruby ${{ matrix.ruby }} tests, minimal=${{ matrix.minimal }}
     steps:
       - uses: actions/checkout@v2
-      - name: Setup Ruby
+      - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
-      - uses: actions/cache@v2
-        with:
-          path: vendor/bundle
-          key: ${{ runner.os }}-${{ matrix.ruby }}-gems-${{ hashFiles('Gemfile', 'frozen_record.gemspec') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.ruby }}-gems-
-      - name: Bundle install
-        run: |
-          gem install bundler
-          bundle install --jobs 4 --retry 3 --path=vendor/bundle
+          bundler-cache: true
       - name: Run tests
         env:
           MINIMAL: ${{ matrix.minimal }}

data/CHANGELOG.md

@@ -0,0 +1,22 @@
+# Unreleased
+
+# v0.25.0
+
+- Disable max_records_scan checks when loading records.
+- Add `FrozenRecord::Base.with_max_record_scan` for more easily allowing larger amount in specific tests.
+
+# v0.24.1
+
+- Fix index selection not applying some restrictions.
+
+# v0.24.0 (yanked)
+
+- Improve index selection and combinaison. Should significantly help with performance in some cases.
+- Implement `max_records_scan` to reject slow queries.
+- Only load `Railtie` integration if `Rails::Railtie` is defined
+- Allow granular fixture unloading
+- Fix a bug affecting older bootsnap versions
+
+# v0.23.0
+
+NO DATA

data/README.md

@@ -3,7 +3,7 @@
 [![Build Status](https://secure.travis-ci.org/byroot/frozen_record.svg)](http://travis-ci.org/byroot/frozen_record)
 [![Gem Version](https://badge.fury.io/rb/frozen_record.svg)](http://badge.fury.io/rb/frozen_record)
 
-ActiveRecord-like interface for **read only** access to static data files.
+Activec Record-like interface for **read only** access to static data files of reasonable size.
 
 ## Installation
 
@@ -21,7 +21,7 @@ Or install it yourself as:
 
 ## Models definition
 
-Just like with ActiveRecord, your models need to inherits from `FrozenRecord::Base`:
+Just like with Active Record, your models need to inherits from `FrozenRecord::Base`:
 
 ```ruby
 class Country < FrozenRecord::Base
@@ -72,7 +72,7 @@ end
 
 ## Query interface
 
-FrozenRecord aim to replicate only modern ActiveRecord querying interface, and only the non "string typed" ones.
+FrozenRecord aim to replicate only modern Active Record querying interface, and only the non "string typed" ones.
 
 e.g
 ```ruby
@@ -154,6 +154,13 @@ Composite index keys are not supported.
 
 The primary key isn't indexed by default.
 
+## Limitations
+
+Frozen Record is not meant to operate or large unindexed datasets.
+
+To ensure that it doesn't happen by accident, you can set `FrozenRecord::Base.max_records_scan = 500` (or whatever limit makes sense to you), in your development and test environments.
+This setting will cause Frozen Record to raise an error if it has to scan more than `max_records_scan` records. This property can also be set on a per model basis.
+
 ## Configuration
 
 ### Reloading
@@ -190,7 +197,7 @@ require 'frozen_record/test_helper'
 
 class CountryTest < ActiveSupport::TestCase
   setup do
-    test_fixtures_base_path = Rails.root.join(%w(test support fixtures))
+    test_fixtures_base_path = Rails.root.join('test/support/fixtures')
     FrozenRecord::TestHelper.load_fixture(Country, test_fixtures_base_path)
   end
 

data/lib/frozen_record/backends/empty.yml

@@ -0,0 +1 @@
+true

data/lib/frozen_record/backends/yaml.rb

@@ -49,7 +49,7 @@ module FrozenRecord
       private
 
       supports_freeze = begin
-        YAML.load_file(File.expand_path('../empty.json', __FILE__), freeze: true)
+        YAML.load_file(File.expand_path('../empty.yml', __FILE__), freeze: true)
       rescue ArgumentError
         false
       end

data/lib/frozen_record/base.rb

@@ -4,6 +4,8 @@ require 'active_support/descendants_tracker'
 require 'frozen_record/backends'
 
 module FrozenRecord
+  SlowQuery = Class.new(StandardError)
+
   class Base
     extend ActiveSupport::DescendantsTracker
     extend ActiveModel::Naming
@@ -15,6 +17,7 @@ module FrozenRecord
 
     class_attribute :base_path, :primary_key, :backend, :auto_reloading, :default_attributes, instance_accessor: false
     class_attribute :index_definitions, instance_accessor: false
+    class_attribute :max_records_scan, instance_accessor: false
     self.index_definitions = {}.freeze
 
     self.primary_key = 'id'
@@ -42,6 +45,14 @@ module FrozenRecord
     end
 
     class << self
+      def with_max_records_scan(value)
+        previous_max_records_scan = max_records_scan
+        self.max_records_scan = value
+        yield
+      ensure
+        self.max_records_scan = previous_max_records_scan
+      end
+
       alias_method :set_default_attributes, :default_attributes=
       private :set_default_attributes
       def default_attributes=(default_attributes)
@@ -145,7 +156,7 @@ module FrozenRecord
           end
           @attributes = list_attributes(records).freeze
           define_attribute_methods(@attributes.to_a)
-          records = records.map { |r| load(r) }.freeze
+          records = with_max_records_scan(nil) { records.map { |r| load(r) }.freeze }
           index_definitions.values.each { |index| index.build(records) }
           records
         end

data/lib/frozen_record/minimal.rb

@@ -23,5 +23,5 @@ module FrozenRecord
     end
   end
 
-  self.deprecated_yaml_erb_backend = true
+  self.deprecated_yaml_erb_backend = false
 end

data/lib/frozen_record/scope.rb

@@ -173,6 +173,8 @@ module FrozenRecord
       sort_records(select_records(@klass.load_records))
     end
 
+    ARRAY_INTERSECTION = Array.method_defined?(:intersection)
+
     def select_records(records)
       return records if @where_values.empty? && @where_not_values.empty?
 
@@ -180,13 +182,30 @@ module FrozenRecord
       indexed_where_values, unindexed_where_values = @where_values.partition { |criteria| indices.key?(criteria.first) }
 
       unless indexed_where_values.empty?
-        attribute, value = indexed_where_values.shift
-        records = indices[attribute].query(value)
-        indexed_where_values.each do |(attribute, value)|
-          records &= indices[attribute].query(value)
+        usable_indexes = indexed_where_values.map { |(attribute, value)| [attribute, value, indices[attribute].query(value)] }
+        usable_indexes.sort_by! { |r| r[2].size }
+        records = usable_indexes.shift.last
+
+        # If the index is 5 times bigger that the current set of records it's not worth doing an array intersection.
+        # The value is somewhat arbitrary and could be adjusted.
+        useless_indexes, usable_indexes = usable_indexes.partition { |_, _, indexed_records| indexed_records.size > records.size * 5 }
+        unindexed_where_values += useless_indexes.map { |a| a.first(2) }
+
+        unless usable_indexes.empty?
+          if ARRAY_INTERSECTION
+            records = records.intersection(*usable_indexes.map(&:last))
+          else
+            usable_indexes.each do |_, _, indexed_records|
+              records &= indexed_records
+            end
+          end
         end
       end
 
+      if @klass.max_records_scan && records.size > @klass.max_records_scan
+        raise SlowQuery, "Scanning #{records.size} records is too slow, the allowed maximum is #{@klass.max_records_scan}. Try to find a better index or consider an alternative storage"
+      end
+
       records.select do |record|
         unindexed_where_values.all? { |attr, matcher| matcher.match?(record[attr]) } &&
         !@where_not_values.any? { |attr, matcher| matcher.match?(record[attr]) }

data/lib/frozen_record/test_helper.rb

@@ -8,9 +8,7 @@ module FrozenRecord
       def load_fixture(model_class, alternate_base_path)
         @cache ||= {}
 
-        unless model_class < FrozenRecord::Base
-          raise ArgumentError, "Model class (#{model_class}) does not inherit from #{FrozenRecord::Base}"
-        end
+        ensure_model_class_is_frozenrecord(model_class)
 
         return if @cache.key?(model_class)
 
@@ -20,20 +18,32 @@ module FrozenRecord
         model_class.load_records(force: true)
       end
 
-      def unload_fixtures
+      def unload_fixture(model_class)
         return unless defined?(@cache) && @cache
 
-        @cache.each do |model_class, old_base_path|
-          model_class.base_path = old_base_path
-          model_class.load_records(force: true)
-        end
+        ensure_model_class_is_frozenrecord(model_class)
+
+        return unless @cache.key?(model_class)
+
+        old_base_path = @cache[model_class]
+        model_class.base_path = old_base_path
+        model_class.load_records(force: true)
 
-        @cache = nil
+        @cache.delete(model_class)
+      end
+
+      def unload_fixtures
+        return unless defined?(@cache) && @cache
+
+        @cache.keys.each { |model_class| unload_fixture(model_class) }
       end
 
       private
 
       def ensure_model_class_is_frozenrecord(model_class)
+        unless model_class < FrozenRecord::Base
+          raise ArgumentError, "Model class (#{model_class}) does not inherit from #{FrozenRecord::Base}"
+        end
       end
     end
   end

data/lib/frozen_record/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FrozenRecord
-  VERSION = '0.22.2'
+  VERSION = '0.25.0'
 end

data/lib/frozen_record.rb

@@ -2,4 +2,4 @@
 
 require 'frozen_record/minimal'
 require 'frozen_record/serialization'
-require 'frozen_record/railtie' if defined?(Rails)
+require 'frozen_record/railtie' if defined?(Rails::Railtie)

data/spec/fixtures/continents.yml.erb

@@ -0,0 +1,9 @@
+---
+- id: 1
+  name: Africa
+
+- id: 2
+  name: Antarctica
+
+- id: 3
+  name: Asia

data/spec/fixtures/prices.yml.erb

@@ -0,0 +1,13 @@
+<% 100.times.map { |i| "plan_#{i}"}.each do |plan_name| %>
+  <% ["base", "online", "retail"].each do |type| %>
+    <% [nil, "USD", "EUR", "GBP", "JPY"].each do |currency| %>
+     <% ["monthly", "annual", "biennial", "triennial"].each do |period| %>
+- plan_name: <%= plan_name %>
+  type: <%= type %>
+  period: <%= period %>
+  currency: <%= currency %>
+  amount: 10.0
+      <% end %>
+    <% end %>
+  <% end %>
+<% end %>

data/spec/fixtures/test_helper/continents.yml.erb

@@ -0,0 +1,3 @@
+---
+- id: 1
+  name: Some continent

data/spec/scope_spec.rb

@@ -474,4 +474,30 @@ describe 'querying' do
 
   end
 
+  context 'when max_records_scan is set' do
+
+    it 'raises on slow queries' do
+      expect {
+        FrozenRecord::Base.with_max_records_scan(1) do
+          Country.where(king: "Louis").to_a
+        end
+      }.to raise_error(FrozenRecord::SlowQuery)
+    end
+
+    it 'is accurate' do
+      FrozenRecord::Base.with_max_records_scan(60) do
+        expect(Price.count).to be == 6_000
+
+        Price.where(plan_name: "plan_24", currency: [nil, "EUR"], period: "monthly", type: "base").each do |price|
+          expect(price.plan_name).to be == "plan_24"
+          unless price.currency.nil?
+            expect(price.currency).to be  == "EUR"
+          end
+          expect(price.period).to be == "monthly"
+          expect(price.type).to be == "base"
+        end
+      end
+    end
+
+  end
 end

data/spec/support/continent.rb

@@ -0,0 +1,2 @@
+class Continent < FrozenRecord::Base
+end

data/spec/support/price.rb

@@ -0,0 +1,13 @@
+class Price < FrozenRecord::Base
+  add_index :plan_name
+  add_index :currency
+end
+
+module Compact
+  class Price < ::Price
+    include FrozenRecord::Compact
+    def self.file_path
+      superclass.file_path
+    end
+  end
+end

data/spec/test_helper_spec.rb

@@ -38,19 +38,35 @@ describe 'test fixture loading' do
     end
   end
 
+  describe '.unload_fixture' do
+    it 'restores the default fixtures for the specified model class' do
+      test_fixtures_base_path = File.join(File.dirname(__FILE__), 'fixtures', 'test_helper')
+
+      FrozenRecord::TestHelper.load_fixture(Continent, test_fixtures_base_path)
+      FrozenRecord::TestHelper.load_fixture(Country, test_fixtures_base_path)
+      FrozenRecord::TestHelper.unload_fixture(Country)
+
+      expect(Continent.count).to be == 1
+      expect(Country.count).to be == 3
+    end
+  end
+
   describe '.unload_fixtures' do
     it 'restores the default fixtures' do
       test_fixtures_base_path = File.join(File.dirname(__FILE__), 'fixtures', 'test_helper')
 
+      FrozenRecord::TestHelper.load_fixture(Continent, test_fixtures_base_path)
       FrozenRecord::TestHelper.load_fixture(Country, test_fixtures_base_path)
       FrozenRecord::TestHelper.unload_fixtures
 
+      expect(Continent.count).to be == 3
       expect(Country.count).to be == 3
     end
 
     it 'does has no effect if no alternate fixtures were loaded' do
       FrozenRecord::TestHelper.unload_fixtures
 
+      expect(Continent.count).to be == 3
       expect(Country.count).to be == 3
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: frozen_record
 version: !ruby/object:Gem::Version
-  version: 0.22.2
+  version: 0.25.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-06 00:00:00.000000000 Z
+date: 2022-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -76,6 +76,7 @@ files:
 - ".github/workflows/main.yml"
 - ".gitignore"
 - ".rspec"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -88,6 +89,7 @@ files:
 - lib/frozen_record.rb
 - lib/frozen_record/backends.rb
 - lib/frozen_record/backends/empty.json
+- lib/frozen_record/backends/empty.yml
 - lib/frozen_record/backends/json.rb
 - lib/frozen_record/backends/yaml.rb
 - lib/frozen_record/base.rb
@@ -101,7 +103,10 @@ files:
 - lib/frozen_record/version.rb
 - spec/fixtures/animals.json
 - spec/fixtures/cars.yml
+- spec/fixtures/continents.yml.erb
 - spec/fixtures/countries.yml.erb
+- spec/fixtures/prices.yml.erb
+- spec/fixtures/test_helper/continents.yml.erb
 - spec/fixtures/test_helper/countries.yml.erb
 - spec/frozen_record_spec.rb
 - spec/scope_spec.rb
@@ -109,7 +114,9 @@ files:
 - spec/support/abstract_model.rb
 - spec/support/animal.rb
 - spec/support/car.rb
+- spec/support/continent.rb
 - spec/support/country.rb
+- spec/support/price.rb
 - spec/test_helper_spec.rb
 homepage: https://github.com/byroot/frozen_record
 licenses:
@@ -130,14 +137,17 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.0.dev
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: ActiveRecord like interface to read only access and query static YAML files
 test_files:
 - spec/fixtures/animals.json
 - spec/fixtures/cars.yml
+- spec/fixtures/continents.yml.erb
 - spec/fixtures/countries.yml.erb
+- spec/fixtures/prices.yml.erb
+- spec/fixtures/test_helper/continents.yml.erb
 - spec/fixtures/test_helper/countries.yml.erb
 - spec/frozen_record_spec.rb
 - spec/scope_spec.rb
@@ -145,5 +155,7 @@ test_files:
 - spec/support/abstract_model.rb
 - spec/support/animal.rb
 - spec/support/car.rb
+- spec/support/continent.rb
 - spec/support/country.rb
+- spec/support/price.rb
 - spec/test_helper_spec.rb