identity_cache 0.4.1 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (94) hide show
  1. checksums.yaml +5 -5
  2. data/.github/probots.yml +2 -0
  3. data/.github/workflows/ci.yml +92 -0
  4. data/.gitignore +2 -0
  5. data/.rubocop.yml +5 -0
  6. data/CAVEATS.md +25 -0
  7. data/CHANGELOG.md +73 -19
  8. data/Gemfile +5 -1
  9. data/LICENSE +1 -1
  10. data/README.md +49 -27
  11. data/Rakefile +14 -5
  12. data/dev.yml +12 -16
  13. data/gemfiles/Gemfile.latest-release +8 -0
  14. data/gemfiles/Gemfile.min-supported +7 -0
  15. data/gemfiles/Gemfile.rails-edge +7 -0
  16. data/identity_cache.gemspec +29 -10
  17. data/lib/identity_cache.rb +78 -51
  18. data/lib/identity_cache/belongs_to_caching.rb +12 -40
  19. data/lib/identity_cache/cache_fetcher.rb +6 -5
  20. data/lib/identity_cache/cache_hash.rb +2 -2
  21. data/lib/identity_cache/cache_invalidation.rb +4 -11
  22. data/lib/identity_cache/cache_key_generation.rb +17 -65
  23. data/lib/identity_cache/cache_key_loader.rb +128 -0
  24. data/lib/identity_cache/cached.rb +7 -0
  25. data/lib/identity_cache/cached/association.rb +87 -0
  26. data/lib/identity_cache/cached/attribute.rb +123 -0
  27. data/lib/identity_cache/cached/attribute_by_multi.rb +37 -0
  28. data/lib/identity_cache/cached/attribute_by_one.rb +88 -0
  29. data/lib/identity_cache/cached/belongs_to.rb +100 -0
  30. data/lib/identity_cache/cached/embedded_fetching.rb +41 -0
  31. data/lib/identity_cache/cached/prefetcher.rb +61 -0
  32. data/lib/identity_cache/cached/primary_index.rb +96 -0
  33. data/lib/identity_cache/cached/recursive/association.rb +109 -0
  34. data/lib/identity_cache/cached/recursive/has_many.rb +9 -0
  35. data/lib/identity_cache/cached/recursive/has_one.rb +9 -0
  36. data/lib/identity_cache/cached/reference/association.rb +16 -0
  37. data/lib/identity_cache/cached/reference/has_many.rb +105 -0
  38. data/lib/identity_cache/cached/reference/has_one.rb +100 -0
  39. data/lib/identity_cache/configuration_dsl.rb +53 -215
  40. data/lib/identity_cache/encoder.rb +95 -0
  41. data/lib/identity_cache/expiry_hook.rb +36 -0
  42. data/lib/identity_cache/fallback_fetcher.rb +2 -1
  43. data/lib/identity_cache/load_strategy/eager.rb +28 -0
  44. data/lib/identity_cache/load_strategy/lazy.rb +71 -0
  45. data/lib/identity_cache/load_strategy/load_request.rb +20 -0
  46. data/lib/identity_cache/load_strategy/multi_load_request.rb +27 -0
  47. data/lib/identity_cache/mem_cache_store_cas.rb +53 -0
  48. data/lib/identity_cache/memoized_cache_proxy.rb +137 -58
  49. data/lib/identity_cache/parent_model_expiration.rb +46 -11
  50. data/lib/identity_cache/query_api.rb +102 -408
  51. data/lib/identity_cache/railtie.rb +8 -0
  52. data/lib/identity_cache/record_not_found.rb +6 -0
  53. data/lib/identity_cache/should_use_cache.rb +1 -0
  54. data/lib/identity_cache/version.rb +3 -2
  55. data/lib/identity_cache/with_primary_index.rb +136 -0
  56. data/lib/identity_cache/without_primary_index.rb +24 -3
  57. data/performance/cache_runner.rb +25 -73
  58. data/performance/cpu.rb +4 -3
  59. data/performance/externals.rb +4 -3
  60. data/performance/profile.rb +6 -5
  61. data/railgun.yml +16 -0
  62. metadata +60 -73
  63. data/.travis.yml +0 -30
  64. data/Gemfile.rails42 +0 -6
  65. data/Gemfile.rails50 +0 -6
  66. data/test/attribute_cache_test.rb +0 -110
  67. data/test/cache_fetch_includes_test.rb +0 -46
  68. data/test/cache_hash_test.rb +0 -14
  69. data/test/cache_invalidation_test.rb +0 -139
  70. data/test/deeply_nested_associated_record_test.rb +0 -19
  71. data/test/denormalized_has_many_test.rb +0 -211
  72. data/test/denormalized_has_one_test.rb +0 -160
  73. data/test/fetch_multi_test.rb +0 -308
  74. data/test/fetch_test.rb +0 -258
  75. data/test/fixtures/serialized_record.mysql2 +0 -0
  76. data/test/fixtures/serialized_record.postgresql +0 -0
  77. data/test/helpers/active_record_objects.rb +0 -106
  78. data/test/helpers/database_connection.rb +0 -72
  79. data/test/helpers/serialization_format.rb +0 -42
  80. data/test/helpers/update_serialization_format.rb +0 -24
  81. data/test/identity_cache_test.rb +0 -29
  82. data/test/index_cache_test.rb +0 -161
  83. data/test/memoized_attributes_test.rb +0 -49
  84. data/test/memoized_cache_proxy_test.rb +0 -107
  85. data/test/normalized_belongs_to_test.rb +0 -107
  86. data/test/normalized_has_many_test.rb +0 -231
  87. data/test/normalized_has_one_test.rb +0 -9
  88. data/test/prefetch_associations_test.rb +0 -364
  89. data/test/readonly_test.rb +0 -109
  90. data/test/recursive_denormalized_has_many_test.rb +0 -131
  91. data/test/save_test.rb +0 -82
  92. data/test/schema_change_test.rb +0 -112
  93. data/test/serialization_format_change_test.rb +0 -16
  94. data/test/test_helper.rb +0 -140
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+ module IdentityCache
3
+ class Railtie < Rails::Railtie
4
+ initializer "identity_cache.setup" do |app|
5
+ app.config.eager_load_namespaces << IdentityCache
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,6 @@
1
+ # frozen_string_literal: true
2
+
3
+ module IdentityCache
4
+ class RecordNotFound < ActiveRecord::RecordNotFound
5
+ end
6
+ end
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module IdentityCache
2
3
  module ShouldUseCache
3
4
  extend ActiveSupport::Concern
@@ -1,4 +1,5 @@
1
+ # frozen_string_literal: true
1
2
  module IdentityCache
2
- VERSION = "0.4.1"
3
- CACHE_VERSION = 6
3
+ VERSION = "1.1.0"
4
+ CACHE_VERSION = 8
4
5
  end
@@ -0,0 +1,136 @@
1
+ # frozen_string_literal: true
2
+ module IdentityCache
3
+ module WithPrimaryIndex
4
+ extend ActiveSupport::Concern
5
+
6
+ include WithoutPrimaryIndex
7
+
8
+ def expire_cache
9
+ expire_primary_index
10
+ super
11
+ end
12
+
13
+ # @api private
14
+ def expire_primary_index # :nodoc:
15
+ self.class.expire_primary_key_cache_index(id)
16
+ end
17
+
18
+ # @api private
19
+ def primary_cache_index_key # :nodoc:
20
+ self.class.cached_primary_index.cache_key(id)
21
+ end
22
+
23
+ module ClassMethods
24
+ # @api private
25
+ def cached_primary_index
26
+ @cached_primary_index ||= Cached::PrimaryIndex.new(self)
27
+ end
28
+
29
+ def primary_cache_index_enabled
30
+ true
31
+ end
32
+
33
+ # Declares a new index in the cache for the class where IdentityCache was
34
+ # included.
35
+ #
36
+ # IdentityCache will add a fetch_by_field1_and_field2_and_...field for every
37
+ # index.
38
+ #
39
+ # == Example:
40
+ #
41
+ # class Product
42
+ # include IdentityCache
43
+ # cache_index :name, :vendor
44
+ # end
45
+ #
46
+ # Will add Product.fetch_by_name_and_vendor
47
+ #
48
+ # == Parameters
49
+ #
50
+ # +fields+ Array of symbols or strings representing the fields in the index
51
+ #
52
+ # == Options
53
+ # * unique: if the index would only have unique values. Default is false
54
+ #
55
+ def cache_index(*fields, unique: false)
56
+ attribute_proc = -> { primary_key }
57
+ cache_attribute_by_alias(attribute_proc, alias_name: :id, by: fields, unique: unique)
58
+
59
+ field_list = fields.join("_and_")
60
+ arg_list = (0...fields.size).collect { |i| "arg#{i}" }.join(',')
61
+
62
+ if unique
63
+ instance_eval(<<-CODE, __FILE__, __LINE__ + 1)
64
+ def fetch_by_#{field_list}(#{arg_list}, includes: nil)
65
+ id = fetch_id_by_#{field_list}(#{arg_list})
66
+ id && fetch_by_id(id, includes: includes)
67
+ end
68
+
69
+ # exception throwing variant
70
+ def fetch_by_#{field_list}!(#{arg_list}, includes: nil)
71
+ fetch_by_#{field_list}(#{arg_list}, includes: includes) or raise IdentityCache::RecordNotFound
72
+ end
73
+ CODE
74
+ else
75
+ instance_eval(<<-CODE, __FILE__, __LINE__ + 1)
76
+ def fetch_by_#{field_list}(#{arg_list}, includes: nil)
77
+ ids = fetch_id_by_#{field_list}(#{arg_list})
78
+ ids.empty? ? ids : fetch_multi(ids, includes: includes)
79
+ end
80
+ CODE
81
+ end
82
+
83
+ if fields.length == 1
84
+ instance_eval(<<-CODE, __FILE__, __LINE__ + 1)
85
+ def fetch_multi_by_#{field_list}(index_values, includes: nil)
86
+ ids = fetch_multi_id_by_#{field_list}(index_values).values.flatten(1)
87
+ return ids if ids.empty?
88
+ fetch_multi(ids, includes: includes)
89
+ end
90
+ CODE
91
+ end
92
+ end
93
+
94
+ # Similar to ActiveRecord::Base#exists? will return true if the id can be
95
+ # found in the cache or in the DB.
96
+ def exists_with_identity_cache?(id)
97
+ !!fetch_by_id(id)
98
+ end
99
+
100
+ # Default fetcher added to the model on inclusion, it behaves like
101
+ # ActiveRecord::Base.where(id: id).first
102
+ def fetch_by_id(id, includes: nil)
103
+ ensure_base_model
104
+ raise_if_scoped
105
+ record = cached_primary_index.fetch(id)
106
+ prefetch_associations(includes, [record]) if record && includes
107
+ record
108
+ end
109
+
110
+ # Default fetcher added to the model on inclusion, it behaves like
111
+ # ActiveRecord::Base.find, but will raise IdentityCache::RecordNotFound
112
+ # if the id is not in the cache.
113
+ def fetch(id, includes: nil)
114
+ fetch_by_id(id, includes: includes) || raise(
115
+ IdentityCache::RecordNotFound, "Couldn't find #{name} with ID=#{id}"
116
+ )
117
+ end
118
+
119
+ # Default fetcher added to the model on inclusion, if behaves like
120
+ # ActiveRecord::Base.find_all_by_id
121
+ def fetch_multi(*ids, includes: nil)
122
+ ensure_base_model
123
+ raise_if_scoped
124
+ ids.flatten!(1)
125
+ records = cached_primary_index.fetch_multi(ids)
126
+ prefetch_associations(includes, records) if includes
127
+ records
128
+ end
129
+
130
+ # Invalidates the primary cache index for the associated record. Will not invalidate cached attributes.
131
+ def expire_primary_key_cache_index(id)
132
+ cached_primary_index.expire(id)
133
+ end
134
+ end
135
+ end
136
+ end
@@ -1,10 +1,31 @@
1
+ # frozen_string_literal: true
1
2
  module IdentityCache
2
3
  module WithoutPrimaryIndex
3
4
  extend ActiveSupport::Concern
4
5
 
5
- included do |base|
6
- base.send(:include, IdentityCache)
7
- base.primary_cache_index_enabled = false
6
+ include ArTransactionChanges
7
+ include IdentityCache::BelongsToCaching
8
+ include IdentityCache::CacheKeyGeneration
9
+ include IdentityCache::ConfigurationDSL
10
+ include IdentityCache::QueryAPI
11
+ include IdentityCache::CacheInvalidation
12
+ include IdentityCache::ShouldUseCache
13
+ include ParentModelExpiration
14
+
15
+ def self.append_features(base) #:nodoc:
16
+ raise AlreadyIncludedError if base.include?(WithoutPrimaryIndex)
17
+ super
18
+ end
19
+
20
+ included do
21
+ class_attribute(:cached_model)
22
+ self.cached_model = self
23
+ end
24
+
25
+ module ClassMethods
26
+ def primary_cache_index_enabled
27
+ false
28
+ end
8
29
  end
9
30
  end
10
31
  end
@@ -1,4 +1,5 @@
1
- $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
1
+ # frozen_string_literal: true
2
+ $LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
2
3
  require 'active_record'
3
4
  require 'active_support/core_ext'
4
5
  require 'active_support/cache'
@@ -6,25 +7,19 @@ require 'identity_cache'
6
7
  require 'memcached_store'
7
8
  require 'active_support/cache/memcached_store'
8
9
 
9
- $memcached_port = 11211
10
- $mysql_port = 3306
11
-
12
10
  require File.dirname(__FILE__) + '/../test/helpers/active_record_objects'
13
11
  require File.dirname(__FILE__) + '/../test/helpers/database_connection'
12
+ require File.dirname(__FILE__) + '/../test/helpers/cache_connection'
14
13
 
15
14
  IdentityCache.logger = Logger.new(nil)
16
- IdentityCache.cache_backend = ActiveSupport::Cache::MemcachedStore.new("localhost:#{$memcached_port}", :support_cas => true)
17
-
18
- if ActiveRecord.gem_version < Gem::Version.new('5') && ActiveRecord::Base.respond_to?(:raise_in_transactional_callbacks=)
19
- ActiveRecord::Base.raise_in_transactional_callbacks = true
20
- end
15
+ CacheConnection.setup
21
16
 
22
17
  def create_record(id)
23
18
  Item.new(id)
24
19
  end
25
20
 
26
21
  def database_ready(count)
27
- Item.where(:id => (1..count)).count == count
22
+ Item.where(id: (1..count)).count == count
28
23
  rescue
29
24
  false
30
25
  end
@@ -42,26 +37,25 @@ def create_database(count)
42
37
  DatabaseConnection.create_tables
43
38
  existing = Item.all
44
39
  (1..count).to_a.each do |i|
45
- unless existing.any? { |e| e.id == i }
46
- a = Item.new
47
- a.id = i
48
- a.associated = AssociatedRecord.new(name: "Associated for #{i}")
49
- a.associated_records
50
- (1..5).each do |j|
51
- a.associated_records << AssociatedRecord.new(name: "Has Many #{j} for #{i}")
52
- a.normalized_associated_records << NormalizedAssociatedRecord.new(name: "Normalized Has Many #{j} for #{i}")
53
- end
54
- a.save
40
+ next if existing.any? { |e| e.id == i }
41
+ a = Item.new
42
+ a.id = i
43
+ a.associated = AssociatedRecord.new(name: "Associated for #{i}")
44
+ a.associated_records
45
+ (1..5).each do |j|
46
+ a.associated_records << AssociatedRecord.new(name: "Has Many #{j} for #{i}")
47
+ a.normalized_associated_records << NormalizedAssociatedRecord.new(name: "Normalized Has Many #{j} for #{i}")
55
48
  end
49
+ a.save
56
50
  end
57
51
  ensure
58
52
  helper.teardown_models
59
53
  end
60
54
 
61
55
  def setup_embedded_associations
62
- Item.cache_has_one :associated
63
- Item.cache_has_many :associated_records, :embed => true
64
- AssociatedRecord.cache_has_many :deeply_associated_records, :embed => true
56
+ Item.cache_has_one(:associated, embed: true)
57
+ Item.cache_has_many(:associated_records, embed: true)
58
+ AssociatedRecord.cache_has_many(:deeply_associated_records, embed: true)
65
59
  end
66
60
 
67
61
  class CacheRunner
@@ -86,7 +80,7 @@ CACHE_RUNNERS = []
86
80
  class FindRunner < CacheRunner
87
81
  def run
88
82
  (1..@count).each do |i|
89
- ::Item.includes(:associated, {:associated_records => :deeply_associated_records}).find(i)
83
+ ::Item.includes(:associated, associated_records: :deeply_associated_records).find(i)
90
84
  end
91
85
  end
92
86
  end
@@ -109,38 +103,16 @@ end
109
103
  module DeletedRunner
110
104
  def prepare
111
105
  super
112
- (1..@count).each {|i| ::Item.find(i).send(:expire_cache) }
113
- end
114
- end
115
-
116
- module ConflictRunner
117
- def prepare
118
- super
119
- records = (1..@count).map {|id| ::Item.find(id) }
120
- orig_resolve_cache_miss = ::Item.method(:resolve_cache_miss)
121
-
122
- ::Item.define_singleton_method(:resolve_cache_miss) do |id|
123
- records[id-1].send(:expire_cache)
124
- orig_resolve_cache_miss.call(id)
125
- end
126
- IdentityCache.cache.clear
127
- end
128
- end
129
-
130
- module DeletedConflictRunner
131
- include ConflictRunner
132
- def prepare
133
- super
134
- (1..@count).each {|i| ::Item.find(i).send(:expire_cache) }
106
+ (1..@count).each { |i| ::Item.find(i).expire_cache }
135
107
  end
136
108
  end
137
109
 
138
110
  class EmbedRunner < CacheRunner
139
111
  def setup_models
140
112
  super
141
- Item.cache_has_one :associated
142
- Item.cache_has_many :associated_records, :embed => true
143
- AssociatedRecord.cache_has_many :deeply_associated_records, :embed => true
113
+ Item.cache_has_one(:associated, embed: true)
114
+ Item.cache_has_many(:associated_records, embed: true)
115
+ AssociatedRecord.cache_has_many(:deeply_associated_records, embed: true)
144
116
  end
145
117
 
146
118
  def run
@@ -167,22 +139,12 @@ class FetchEmbedDeletedRunner < EmbedRunner
167
139
  end
168
140
  CACHE_RUNNERS << FetchEmbedDeletedRunner
169
141
 
170
- class FetchEmbedConflictRunner < EmbedRunner
171
- include ConflictRunner
172
- end
173
- CACHE_RUNNERS << FetchEmbedConflictRunner
174
-
175
- class FetchEmbedDeletedConflictRunner < EmbedRunner
176
- include DeletedConflictRunner
177
- end
178
- CACHE_RUNNERS << FetchEmbedDeletedConflictRunner
179
-
180
142
  class NormalizedRunner < CacheRunner
181
143
  def setup_models
182
144
  super
183
- Item.cache_has_one :associated # :embed => false isn't supported
184
- Item.cache_has_many :associated_records, :embed => :ids
185
- AssociatedRecord.cache_has_many :deeply_associated_records, :embed => :ids
145
+ Item.cache_has_one(:associated, embed: :id)
146
+ Item.cache_has_many(:associated_records, embed: :ids)
147
+ AssociatedRecord.cache_has_many(:deeply_associated_records, embed: :ids)
186
148
  end
187
149
 
188
150
  def run
@@ -210,13 +172,3 @@ class FetchNormalizedDeletedRunner < NormalizedRunner
210
172
  include DeletedRunner
211
173
  end
212
174
  CACHE_RUNNERS << FetchNormalizedDeletedRunner
213
-
214
- class FetchNormalizedConflictRunner < EmbedRunner
215
- include ConflictRunner
216
- end
217
- CACHE_RUNNERS << FetchNormalizedConflictRunner
218
-
219
- class FetchNormalizedDeletedConflictRunner < EmbedRunner
220
- include DeletedConflictRunner
221
- end
222
- CACHE_RUNNERS << FetchNormalizedDeletedConflictRunner
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  require 'rubygems'
2
3
  require 'benchmark'
3
4
 
@@ -19,16 +20,16 @@ ensure
19
20
  obj.cleanup
20
21
  end
21
22
 
22
- def benchmark(runners, label_width=0)
23
+ def benchmark(runners, label_width = 0)
23
24
  IdentityCache.cache.clear
24
25
  runners.each do |runner|
25
- print "#{runner.name}: ".ljust(label_width)
26
+ print("#{runner.name}: ".ljust(label_width))
26
27
  puts run(runner.new(RUNS))
27
28
  end
28
29
  end
29
30
 
30
31
  def bmbm(runners)
31
- label_width = runners.map{ |r| r.name.size }.max + 2
32
+ label_width = runners.map { |r| r.name.size }.max + 2
32
33
  width = label_width + Benchmark::CAPTION.size
33
34
 
34
35
  puts 'Rehearsal: '.ljust(width, '-')
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  require 'rubygems'
2
3
  require 'benchmark'
3
4
  require 'ruby-prof'
@@ -7,8 +8,8 @@ require_relative 'cache_runner'
7
8
  RUNS = 1000
8
9
  RubyProf.measure_mode = RubyProf::CPU_TIME
9
10
 
10
- EXTERNALS = {"Memcache" => ["MemCache#set", "MemCache#get"],
11
- "Database" => ["Mysql2::Client#query"]}
11
+ EXTERNALS = { "Memcache" => ["MemCache#set", "MemCache#get"],
12
+ "Database" => ["Mysql2::Client#query"] }
12
13
 
13
14
  def run(obj)
14
15
  obj.prepare
@@ -26,7 +27,7 @@ def count_externals(results)
26
27
  count = {}
27
28
  results.split(/\n/).each do |line|
28
29
  fields = line.split
29
- if ext = EXTERNALS.detect { |e| e[1].any? { |method| method == fields[-1] } }
30
+ if (ext = EXTERNALS.detect { |e| e[1].any? { |method| method == fields[-1] } })
30
31
  count[ext[0]] ||= 0
31
32
  count[ext[0]] += fields[-2].to_i
32
33
  end
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  require 'rubygems'
2
3
  require 'benchmark'
3
4
  require 'stackprof'
@@ -21,15 +22,15 @@ end
21
22
 
22
23
  create_database(RUNS)
23
24
 
24
- if runner_name = ENV['RUNNER']
25
- if runner = CACHE_RUNNERS.find{ |r| r.name == runner_name }
25
+ if (runner_name = ENV['RUNNER'])
26
+ if (runner = CACHE_RUNNERS.find { |r| r.name == runner_name })
26
27
  run(runner.new(RUNS), filename: ENV['FILENAME'])
27
28
  else
28
29
  puts "Couldn't find cache runner #{runner_name.inspect}"
29
- exit 1
30
+ exit(1)
30
31
  end
31
32
  else
32
- CACHE_RUNNERS.each do |runner|
33
- run(runner.new(RUNS))
33
+ CACHE_RUNNERS.each do |cache_runner|
34
+ run(cache_runner.new(RUNS))
34
35
  end
35
36
  end
@@ -0,0 +1,16 @@
1
+ # https://dev-accel.shopify.io/dev/railgun/Railgun-Config
2
+ name: identity-cache
3
+
4
+ vm:
5
+ image: /opt/dev/misc/railgun-images/default
6
+ ip_address: 192.168.64.98
7
+ memory: 1G
8
+ cores: 2
9
+
10
+ volumes:
11
+ root: 1G
12
+
13
+ services:
14
+ - mysql
15
+ - postgresql
16
+ - memcached