identity_cache 0.4.1 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. checksums.yaml +5 -5
  2. data/.github/probots.yml +2 -0
  3. data/.github/workflows/ci.yml +92 -0
  4. data/.gitignore +2 -0
  5. data/.rubocop.yml +5 -0
  6. data/CAVEATS.md +25 -0
  7. data/CHANGELOG.md +73 -19
  8. data/Gemfile +5 -1
  9. data/LICENSE +1 -1
  10. data/README.md +49 -27
  11. data/Rakefile +14 -5
  12. data/dev.yml +12 -16
  13. data/gemfiles/Gemfile.latest-release +8 -0
  14. data/gemfiles/Gemfile.min-supported +7 -0
  15. data/gemfiles/Gemfile.rails-edge +7 -0
  16. data/identity_cache.gemspec +29 -10
  17. data/lib/identity_cache.rb +78 -51
  18. data/lib/identity_cache/belongs_to_caching.rb +12 -40
  19. data/lib/identity_cache/cache_fetcher.rb +6 -5
  20. data/lib/identity_cache/cache_hash.rb +2 -2
  21. data/lib/identity_cache/cache_invalidation.rb +4 -11
  22. data/lib/identity_cache/cache_key_generation.rb +17 -65
  23. data/lib/identity_cache/cache_key_loader.rb +128 -0
  24. data/lib/identity_cache/cached.rb +7 -0
  25. data/lib/identity_cache/cached/association.rb +87 -0
  26. data/lib/identity_cache/cached/attribute.rb +123 -0
  27. data/lib/identity_cache/cached/attribute_by_multi.rb +37 -0
  28. data/lib/identity_cache/cached/attribute_by_one.rb +88 -0
  29. data/lib/identity_cache/cached/belongs_to.rb +100 -0
  30. data/lib/identity_cache/cached/embedded_fetching.rb +41 -0
  31. data/lib/identity_cache/cached/prefetcher.rb +61 -0
  32. data/lib/identity_cache/cached/primary_index.rb +96 -0
  33. data/lib/identity_cache/cached/recursive/association.rb +109 -0
  34. data/lib/identity_cache/cached/recursive/has_many.rb +9 -0
  35. data/lib/identity_cache/cached/recursive/has_one.rb +9 -0
  36. data/lib/identity_cache/cached/reference/association.rb +16 -0
  37. data/lib/identity_cache/cached/reference/has_many.rb +105 -0
  38. data/lib/identity_cache/cached/reference/has_one.rb +100 -0
  39. data/lib/identity_cache/configuration_dsl.rb +53 -215
  40. data/lib/identity_cache/encoder.rb +95 -0
  41. data/lib/identity_cache/expiry_hook.rb +36 -0
  42. data/lib/identity_cache/fallback_fetcher.rb +2 -1
  43. data/lib/identity_cache/load_strategy/eager.rb +28 -0
  44. data/lib/identity_cache/load_strategy/lazy.rb +71 -0
  45. data/lib/identity_cache/load_strategy/load_request.rb +20 -0
  46. data/lib/identity_cache/load_strategy/multi_load_request.rb +27 -0
  47. data/lib/identity_cache/mem_cache_store_cas.rb +53 -0
  48. data/lib/identity_cache/memoized_cache_proxy.rb +137 -58
  49. data/lib/identity_cache/parent_model_expiration.rb +46 -11
  50. data/lib/identity_cache/query_api.rb +102 -408
  51. data/lib/identity_cache/railtie.rb +8 -0
  52. data/lib/identity_cache/record_not_found.rb +6 -0
  53. data/lib/identity_cache/should_use_cache.rb +1 -0
  54. data/lib/identity_cache/version.rb +3 -2
  55. data/lib/identity_cache/with_primary_index.rb +136 -0
  56. data/lib/identity_cache/without_primary_index.rb +24 -3
  57. data/performance/cache_runner.rb +25 -73
  58. data/performance/cpu.rb +4 -3
  59. data/performance/externals.rb +4 -3
  60. data/performance/profile.rb +6 -5
  61. data/railgun.yml +16 -0
  62. metadata +60 -73
  63. data/.travis.yml +0 -30
  64. data/Gemfile.rails42 +0 -6
  65. data/Gemfile.rails50 +0 -6
  66. data/test/attribute_cache_test.rb +0 -110
  67. data/test/cache_fetch_includes_test.rb +0 -46
  68. data/test/cache_hash_test.rb +0 -14
  69. data/test/cache_invalidation_test.rb +0 -139
  70. data/test/deeply_nested_associated_record_test.rb +0 -19
  71. data/test/denormalized_has_many_test.rb +0 -211
  72. data/test/denormalized_has_one_test.rb +0 -160
  73. data/test/fetch_multi_test.rb +0 -308
  74. data/test/fetch_test.rb +0 -258
  75. data/test/fixtures/serialized_record.mysql2 +0 -0
  76. data/test/fixtures/serialized_record.postgresql +0 -0
  77. data/test/helpers/active_record_objects.rb +0 -106
  78. data/test/helpers/database_connection.rb +0 -72
  79. data/test/helpers/serialization_format.rb +0 -42
  80. data/test/helpers/update_serialization_format.rb +0 -24
  81. data/test/identity_cache_test.rb +0 -29
  82. data/test/index_cache_test.rb +0 -161
  83. data/test/memoized_attributes_test.rb +0 -49
  84. data/test/memoized_cache_proxy_test.rb +0 -107
  85. data/test/normalized_belongs_to_test.rb +0 -107
  86. data/test/normalized_has_many_test.rb +0 -231
  87. data/test/normalized_has_one_test.rb +0 -9
  88. data/test/prefetch_associations_test.rb +0 -364
  89. data/test/readonly_test.rb +0 -109
  90. data/test/recursive_denormalized_has_many_test.rb +0 -131
  91. data/test/save_test.rb +0 -82
  92. data/test/schema_change_test.rb +0 -112
  93. data/test/serialization_format_change_test.rb +0 -16
  94. data/test/test_helper.rb +0 -140
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module IdentityCache
4
+ module Cached
5
+ class AttributeByMulti < Attribute
6
+ def build
7
+ cached_attribute = self
8
+
9
+ model.define_singleton_method(:"fetch_#{fetch_method_suffix}") do |*key_values|
10
+ raise_if_scoped
11
+ cached_attribute.fetch(key_values)
12
+ end
13
+ end
14
+
15
+ private
16
+
17
+ # Attribute method overrides
18
+
19
+ def cast_db_key(key_values)
20
+ field_types.each_with_index do |type, i|
21
+ key_values[i] = type.cast(key_values[i])
22
+ end
23
+ key_values
24
+ end
25
+
26
+ def unhashed_values_cache_key_string(key_values)
27
+ key_values.map { |v| v.try!(:to_s).inspect }.join('/')
28
+ end
29
+
30
+ def load_from_db_where_conditions(key_values)
31
+ Hash[key_fields.zip(key_values)]
32
+ end
33
+
34
+ alias_method :cache_key_from_key_values, :cache_key
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,88 @@
1
+ # frozen_string_literal: true
2
+
3
+ module IdentityCache
4
+ module Cached
5
+ class AttributeByOne < Attribute
6
+ attr_reader :key_field
7
+
8
+ def initialize(*)
9
+ super
10
+ @key_field = key_fields.first
11
+ end
12
+
13
+ def build
14
+ cached_attribute = self
15
+
16
+ model.define_singleton_method(:"fetch_#{fetch_method_suffix}") do |key|
17
+ raise_if_scoped
18
+ cached_attribute.fetch(key)
19
+ end
20
+
21
+ model.define_singleton_method(:"fetch_multi_#{fetch_method_suffix}") do |keys|
22
+ raise_if_scoped
23
+ cached_attribute.fetch_multi(keys)
24
+ end
25
+ end
26
+
27
+ def fetch_multi(keys)
28
+ keys = keys.map { |key| cast_db_key(key) }
29
+
30
+ unless model.should_use_cache?
31
+ return load_multi_from_db(keys)
32
+ end
33
+
34
+ unordered_hash = CacheKeyLoader.load_multi(self, keys)
35
+
36
+ # Calling `values` on the result is expected to return the values in the same order as their
37
+ # corresponding keys. The fetch_multi_by_#{field_list} generated methods depend on this.
38
+ ordered_hash = {}
39
+ keys.each { |key| ordered_hash[key] = unordered_hash.fetch(key) }
40
+ ordered_hash
41
+ end
42
+
43
+ def load_multi_from_db(keys)
44
+ rows = model.reorder(nil).where(load_from_db_where_conditions(keys)).pluck(key_field, attribute)
45
+ result = {}
46
+ default = unique ? nil : []
47
+ keys.each do |index_value|
48
+ result[index_value] = default.try!(:dup)
49
+ end
50
+ if unique
51
+ rows.each do |index_value, attribute_value|
52
+ result[index_value] = attribute_value
53
+ end
54
+ else
55
+ rows.each do |index_value, attribute_value|
56
+ result[index_value] << attribute_value
57
+ end
58
+ end
59
+ result
60
+ end
61
+
62
+ def cache_encode(db_value)
63
+ db_value
64
+ end
65
+ alias_method :cache_decode, :cache_encode
66
+
67
+ private
68
+
69
+ # Attribute method overrides
70
+
71
+ def cast_db_key(key)
72
+ field_types.first.cast(key)
73
+ end
74
+
75
+ def unhashed_values_cache_key_string(key)
76
+ key.try!(:to_s).inspect
77
+ end
78
+
79
+ def load_from_db_where_conditions(key_values)
80
+ { key_field => key_values }
81
+ end
82
+
83
+ def cache_key_from_key_values(key_values)
84
+ cache_key(key_values.first)
85
+ end
86
+ end
87
+ end
88
+ end
@@ -0,0 +1,100 @@
1
+ # frozen_string_literal: true
2
+ module IdentityCache
3
+ module Cached
4
+ class BelongsTo < Association # :nodoc:
5
+ attr_reader :records_variable_name
6
+
7
+ def build
8
+ reflection.active_record.class_eval(<<-RUBY, __FILE__, __LINE__ + 1)
9
+ def #{cached_accessor_name}
10
+ association_klass = association(:#{name}).klass
11
+ if association_klass.should_use_cache? && #{reflection.foreign_key}.present? && !association(:#{name}).loaded?
12
+ if defined?(#{records_variable_name})
13
+ #{records_variable_name}
14
+ else
15
+ #{records_variable_name} = association_klass.fetch_by_id(#{reflection.foreign_key})
16
+ end
17
+ else
18
+ #{name}
19
+ end
20
+ end
21
+ RUBY
22
+ end
23
+
24
+ def clear(record)
25
+ if record.instance_variable_defined?(records_variable_name)
26
+ record.remove_instance_variable(records_variable_name)
27
+ end
28
+ end
29
+
30
+ def write(owner_record, associated_record)
31
+ owner_record.instance_variable_set(records_variable_name, associated_record)
32
+ end
33
+
34
+ def fetch(records)
35
+ fetch_async(LoadStrategy::Eager, records) { |associated_records| associated_records }
36
+ end
37
+
38
+ def fetch_async(load_strategy, records)
39
+ if reflection.polymorphic?
40
+ type_fetcher_to_db_ids_hash = {}
41
+
42
+ records.each do |owner_record|
43
+ associated_id = owner_record.send(reflection.foreign_key)
44
+ next unless associated_id && !owner_record.instance_variable_defined?(records_variable_name)
45
+ foreign_type_fetcher = Object.const_get(
46
+ owner_record.send(reflection.foreign_type)
47
+ ).cached_model.cached_primary_index
48
+ db_ids = type_fetcher_to_db_ids_hash[foreign_type_fetcher] ||= []
49
+ db_ids << associated_id
50
+ end
51
+
52
+ load_strategy.load_batch(type_fetcher_to_db_ids_hash) do |batch_load_result|
53
+ batch_records = []
54
+
55
+ records.each do |owner_record|
56
+ associated_id = owner_record.send(reflection.foreign_key)
57
+ next unless associated_id && !owner_record.instance_variable_defined?(records_variable_name)
58
+ foreign_type_fetcher = Object.const_get(
59
+ owner_record.send(reflection.foreign_type)
60
+ ).cached_model.cached_primary_index
61
+
62
+ associated_record = batch_load_result.fetch(foreign_type_fetcher).fetch(associated_id)
63
+ batch_records << owner_record
64
+ write(owner_record, associated_record)
65
+ end
66
+
67
+ yield batch_records
68
+ end
69
+ else
70
+ ids_to_owner_record = records.each_with_object({}) do |owner_record, hash|
71
+ associated_id = owner_record.send(reflection.foreign_key)
72
+ if associated_id && !owner_record.instance_variable_defined?(records_variable_name)
73
+ hash[associated_id] = owner_record
74
+ end
75
+ end
76
+
77
+ load_strategy.load_multi(
78
+ reflection.klass.cached_primary_index,
79
+ ids_to_owner_record.keys
80
+ ) do |associated_records_by_id|
81
+ associated_records_by_id.each do |id, associated_record|
82
+ owner_record = ids_to_owner_record.fetch(id)
83
+ write(owner_record, associated_record)
84
+ end
85
+
86
+ yield associated_records_by_id.values.compact
87
+ end
88
+ end
89
+ end
90
+
91
+ def embedded_recursively?
92
+ false
93
+ end
94
+
95
+ def embedded_by_reference?
96
+ false
97
+ end
98
+ end
99
+ end
100
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+ module IdentityCache
3
+ module Cached
4
+ module EmbeddedFetching
5
+ private
6
+
7
+ def fetch_embedded(records)
8
+ fetch_embedded_async(LoadStrategy::Eager, records) {}
9
+ end
10
+
11
+ def fetch_embedded_async(load_strategy, records)
12
+ return yield if embedded_fetched?(records)
13
+
14
+ klass = reflection.active_record
15
+ cached_associations = klass.send(:embedded_associations)
16
+
17
+ return yield if cached_associations.empty?
18
+
19
+ return yield unless klass.primary_cache_index_enabled
20
+
21
+ load_strategy.load_multi(klass.cached_primary_index, records.map(&:id)) do |cached_records_by_id|
22
+ cached_associations.each_value do |cached_association|
23
+ records.each do |record|
24
+ next unless (cached_record = cached_records_by_id[record.id])
25
+ cached_value = cached_association.read(cached_record)
26
+ cached_association.write(record, cached_value)
27
+ end
28
+ end
29
+
30
+ yield
31
+ end
32
+ end
33
+
34
+ def embedded_fetched?(records)
35
+ # NOTE: Assume all records are the same, so just check the first one.
36
+ record = records.first
37
+ record.association(name).loaded? || record.instance_variable_defined?(records_variable_name)
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,61 @@
1
+ # frozen_string_literal: true
2
+
3
+ module IdentityCache
4
+ module Cached
5
+ module Prefetcher
6
+ ASSOCIATION_FETCH_EVENT = "association_fetch.identity_cache"
7
+
8
+ class << self
9
+ def prefetch(klass, associations, records, load_strategy: LoadStrategy::Eager)
10
+ return if (records = records.to_a).empty?
11
+
12
+ case associations
13
+ when Symbol
14
+ fetch_association(load_strategy, klass, associations, records) {}
15
+ when Array
16
+ load_strategy.lazy_load do |lazy_loader|
17
+ associations.each do |association|
18
+ prefetch(klass, association, records, load_strategy: lazy_loader)
19
+ end
20
+ end
21
+ when Hash
22
+ load_strategy.lazy_load do |lazy_loader|
23
+ associations.each do |association, sub_associations|
24
+ fetch_association(lazy_loader, klass, association, records) do |next_level_records|
25
+ if sub_associations.present?
26
+ association_class = klass.reflect_on_association(association).klass
27
+ prefetch(association_class, sub_associations, next_level_records, load_strategy: lazy_loader)
28
+ end
29
+ end
30
+ end
31
+ end
32
+ else
33
+ raise TypeError, "Invalid associations class #{associations.class}"
34
+ end
35
+ end
36
+
37
+ private
38
+
39
+ def fetch_association(load_strategy, klass, association, records, &block)
40
+ unless klass.should_use_cache?
41
+ preload_records(records, association)
42
+ return yield
43
+ end
44
+
45
+ cached_association = klass.cached_association(association)
46
+ cached_association.fetch_async(load_strategy, records, &block)
47
+ end
48
+
49
+ if ActiveRecord.gem_version < Gem::Version.new("6.2.0.alpha")
50
+ def preload_records(records, association)
51
+ ActiveRecord::Associations::Preloader.new.preload(records, association)
52
+ end
53
+ else
54
+ def preload_records(records, association)
55
+ ActiveRecord::Associations::Preloader.new(records: records, associations: association).call
56
+ end
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,96 @@
1
+ # frozen_string_literal: true
2
+
3
+ module IdentityCache
4
+ module Cached
5
+ class PrimaryIndex
6
+ attr_reader :model
7
+
8
+ def initialize(model)
9
+ @model = model
10
+ end
11
+
12
+ def fetch(id)
13
+ id = cast_id(id)
14
+ return unless id
15
+ record = if model.should_use_cache?
16
+ object = CacheKeyLoader.load(self, id)
17
+ if object && object.id != id
18
+ IdentityCache.logger.error(
19
+ <<~MSG.squish
20
+ [IDC id mismatch] fetch_by_id_requested=#{id}
21
+ fetch_by_id_got=#{object.id}
22
+ for #{object.inspect[(0..100)]}
23
+ MSG
24
+ )
25
+ end
26
+ object
27
+ else
28
+ load_one_from_db(id)
29
+ end
30
+ record
31
+ end
32
+
33
+ def fetch_multi(ids)
34
+ ids.map! { |id| cast_id(id) }.compact!
35
+ id_to_record_hash = if model.should_use_cache?
36
+ id_to_record_hash = CacheKeyLoader.load_multi(self, ids)
37
+ else
38
+ load_multi_from_db(ids)
39
+ end
40
+ records = ids.map { |id| id_to_record_hash[id] }
41
+ records.compact!
42
+ records
43
+ end
44
+
45
+ def expire(id)
46
+ id = cast_id(id)
47
+ IdentityCache.cache.delete(cache_key(id))
48
+ end
49
+
50
+ def cache_key(id)
51
+ "#{model.rails_cache_key_namespace}#{cache_key_prefix}#{id}"
52
+ end
53
+
54
+ def load_one_from_db(id)
55
+ record = build_query(id).take
56
+ model.send(:setup_embedded_associations_on_miss, [record]) if record
57
+ record
58
+ end
59
+
60
+ def load_multi_from_db(ids)
61
+ return {} if ids.empty?
62
+
63
+ records = build_query(ids).to_a
64
+ model.send(:setup_embedded_associations_on_miss, records)
65
+ records.index_by(&:id)
66
+ end
67
+
68
+ def cache_encode(record)
69
+ Encoder.encode(record)
70
+ end
71
+
72
+ def cache_decode(cache_value)
73
+ Encoder.decode(cache_value, model)
74
+ end
75
+
76
+ private
77
+
78
+ def cast_id(id)
79
+ model.type_for_attribute(model.primary_key).cast(id)
80
+ end
81
+
82
+ def id_column
83
+ @id_column ||= model.columns.detect { |c| c.name == model.primary_key }
84
+ end
85
+
86
+ def build_query(id_or_ids)
87
+ model.where(model.primary_key => id_or_ids).includes(model.send(:cache_fetch_includes))
88
+ end
89
+
90
+ def cache_key_prefix
91
+ @cache_key_prefix ||= "blob:#{model.base_class.name}:" \
92
+ "#{IdentityCache::CacheKeyGeneration.denormalized_schema_hash(model)}:"
93
+ end
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,109 @@
1
+ # frozen_string_literal: true
2
+ module IdentityCache
3
+ module Cached
4
+ module Recursive
5
+ class Association < Cached::Association # :nodoc:
6
+ def initialize(name, reflection:)
7
+ super
8
+ @dehydrated_variable_name = :"@dehydrated_#{name}"
9
+ end
10
+
11
+ attr_reader :dehydrated_variable_name
12
+
13
+ def build
14
+ cached_association = self
15
+
16
+ model = reflection.active_record
17
+ model.define_method(cached_accessor_name) do
18
+ cached_association.read(self)
19
+ end
20
+
21
+ ParentModelExpiration.add_parent_expiry_hook(self)
22
+ end
23
+
24
+ def read(record)
25
+ assoc = record.association(name)
26
+
27
+ if assoc.klass.should_use_cache? && !assoc.loaded? && assoc.target.blank?
28
+ if record.instance_variable_defined?(records_variable_name)
29
+ record.instance_variable_get(records_variable_name)
30
+ elsif record.instance_variable_defined?(dehydrated_variable_name)
31
+ dehydrated_target = record.instance_variable_get(dehydrated_variable_name)
32
+ association_target = hydrate_association_target(assoc.klass, dehydrated_target)
33
+ record.remove_instance_variable(dehydrated_variable_name)
34
+ set_with_inverse(record, association_target)
35
+ else
36
+ assoc.load_target
37
+ end
38
+ else
39
+ assoc.load_target
40
+ end
41
+ end
42
+
43
+ def write(record, association_target)
44
+ record.instance_variable_set(records_variable_name, association_target)
45
+ end
46
+
47
+ def set_with_inverse(record, association_target)
48
+ set_inverse(record, association_target)
49
+ write(record, association_target)
50
+ end
51
+
52
+ def clear(record)
53
+ if record.instance_variable_defined?(records_variable_name)
54
+ record.remove_instance_variable(records_variable_name)
55
+ end
56
+ end
57
+
58
+ def fetch(records)
59
+ fetch_async(LoadStrategy::Eager, records) { |child_records| child_records }
60
+ end
61
+
62
+ def fetch_async(load_strategy, records)
63
+ fetch_embedded_async(load_strategy, records) do
64
+ yield records.flat_map(&cached_accessor_name).tap(&:compact!)
65
+ end
66
+ end
67
+
68
+ def embedded_by_reference?
69
+ false
70
+ end
71
+
72
+ def embedded_recursively?
73
+ true
74
+ end
75
+
76
+ private
77
+
78
+ def set_inverse(record, association_target)
79
+ return if association_target.nil?
80
+ associated_class = reflection.klass
81
+ inverse_cached_association = associated_class.cached_belongs_tos[inverse_name]
82
+ return unless inverse_cached_association
83
+
84
+ if association_target.is_a?(Array)
85
+ association_target.each do |child_record|
86
+ inverse_cached_association.write(child_record, record)
87
+ end
88
+ else
89
+ inverse_cached_association.write(association_target, record)
90
+ end
91
+ end
92
+
93
+ def hydrate_association_target(associated_class, dehydrated_value)
94
+ dehydrated_value = IdentityCache.unmap_cached_nil_for(dehydrated_value)
95
+ if dehydrated_value.is_a?(Array)
96
+ dehydrated_value.map { |coder| Encoder.decode(coder, associated_class) }
97
+ else
98
+ Encoder.decode(dehydrated_value, associated_class)
99
+ end
100
+ end
101
+
102
+ def embedded_fetched?(records)
103
+ record = records.first
104
+ super || record.instance_variable_defined?(dehydrated_variable_name)
105
+ end
106
+ end
107
+ end
108
+ end
109
+ end