upkeep-rails 0.1.9 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +105 -195
- data/docs/drafts/turbo-streams-is-cache-invalidation-you-write-by-hand.md +240 -0
- data/docs/handoffs/_archive/2026-06-10-main.md +47 -0
- data/docs/how-it-works.md +8 -0
- data/lib/generators/upkeep/install/install_generator.rb +59 -0
- data/lib/generators/upkeep/install/templates/subscription.js +6 -5
- data/lib/generators/upkeep/install/templates/upkeep.rb +8 -7
- data/lib/upkeep/delivery/turbo_streams.rb +40 -15
- data/lib/upkeep/dependencies.rb +55 -5
- data/lib/upkeep/invalidation/planner.rb +48 -10
- data/lib/upkeep/rails/cable/channel.rb +27 -5
- data/lib/upkeep/rails/cable/subscriber_identity.rb +21 -1
- data/lib/upkeep/rails/client_subscription.rb +12 -12
- data/lib/upkeep/rails/cluster_guard.rb +57 -0
- data/lib/upkeep/rails/configuration.rb +9 -16
- data/lib/upkeep/rails/controller_runtime.rb +17 -0
- data/lib/upkeep/rails/railtie.rb +1 -10
- data/lib/upkeep/rails/testing.rb +1 -1
- data/lib/upkeep/rails.rb +58 -17
- data/lib/upkeep/runtime.rb +39 -2
- data/lib/upkeep/shared_streams.rb +17 -3
- data/lib/upkeep/subscriptions/active_record_store.rb +53 -62
- data/lib/upkeep/subscriptions/active_record_subscription_persistence.rb +6 -3
- data/lib/upkeep/subscriptions/active_registry.rb +0 -7
- data/lib/upkeep/subscriptions/base_store.rb +106 -0
- data/lib/upkeep/subscriptions/layered_reverse_index.rb +9 -13
- data/lib/upkeep/subscriptions/lookup_instrumentation.rb +32 -0
- data/lib/upkeep/subscriptions/persistent_reverse_index.rb +4 -1
- data/lib/upkeep/subscriptions/store.rb +38 -64
- data/lib/upkeep/version.rb +1 -1
- data/upkeep-rails.gemspec +0 -1
- metadata +7 -24
- data/lib/upkeep/rails/delivery_job.rb +0 -29
- data/lib/upkeep/subscriptions/async_durable_writer.rb +0 -131
data/lib/upkeep/rails.rb
CHANGED
|
@@ -4,8 +4,8 @@ require "active_support/notifications"
|
|
|
4
4
|
require_relative "capture/request"
|
|
5
5
|
require_relative "subscriptions/registrar"
|
|
6
6
|
require_relative "rails/configuration"
|
|
7
|
+
require_relative "rails/cluster_guard"
|
|
7
8
|
require_relative "rails/activation_token"
|
|
8
|
-
require_relative "rails/delivery_job"
|
|
9
9
|
require_relative "rails/replay"
|
|
10
10
|
require_relative "rails/action_view_capture"
|
|
11
11
|
require_relative "rails/cable"
|
|
@@ -99,8 +99,8 @@ module Upkeep
|
|
|
99
99
|
nil
|
|
100
100
|
end
|
|
101
101
|
|
|
102
|
-
# Dispatches committed application changes
|
|
103
|
-
#
|
|
102
|
+
# Dispatches committed application changes onto the in-process delivery
|
|
103
|
+
# dispatcher (or synchronously when config.deliver_inline is set).
|
|
104
104
|
#
|
|
105
105
|
# ControllerRuntime calls this automatically after non-GET actions. Apps
|
|
106
106
|
# usually should not call it from controllers or models.
|
|
@@ -122,9 +122,9 @@ module Upkeep
|
|
|
122
122
|
# Delivers committed application changes immediately in the current
|
|
123
123
|
# process.
|
|
124
124
|
#
|
|
125
|
-
# This is used by
|
|
126
|
-
#
|
|
127
|
-
#
|
|
125
|
+
# This is used by inline delivery (config.deliver_inline). Tests that
|
|
126
|
+
# need deterministic async delivery should use Upkeep::Rails::Testing
|
|
127
|
+
# instead of calling this directly.
|
|
128
128
|
#
|
|
129
129
|
# @param changes [Array<#to_h>] change events to deliver. Defaults to the
|
|
130
130
|
# current runtime change log.
|
|
@@ -142,6 +142,7 @@ module Upkeep
|
|
|
142
142
|
return true unless configuration.enabled
|
|
143
143
|
|
|
144
144
|
validate_subscription_store!(environment: environment)
|
|
145
|
+
validate_cluster_safety!(environment: environment)
|
|
145
146
|
true
|
|
146
147
|
end
|
|
147
148
|
|
|
@@ -160,20 +161,15 @@ module Upkeep
|
|
|
160
161
|
|
|
161
162
|
def dispatch_changes(changes)
|
|
162
163
|
payload = {
|
|
163
|
-
|
|
164
|
-
queue: configuration.delivery_queue,
|
|
164
|
+
inline: configuration.deliver_inline,
|
|
165
165
|
change_count: changes.size
|
|
166
166
|
}
|
|
167
167
|
|
|
168
168
|
ActiveSupport::Notifications.instrument(DELIVERY_ENQUEUE, payload) do
|
|
169
|
-
|
|
170
|
-
when :active_job
|
|
171
|
-
DeliveryJob.perform_later(changes)
|
|
172
|
-
Delivery::Transport::DispatchReport.new([])
|
|
173
|
-
when :async
|
|
174
|
-
delivery_dispatcher.enqueue(changes)
|
|
175
|
-
when :inline
|
|
169
|
+
if configuration.deliver_inline
|
|
176
170
|
deliver_changes_now!(changes)
|
|
171
|
+
else
|
|
172
|
+
delivery_dispatcher.enqueue(changes)
|
|
177
173
|
end
|
|
178
174
|
end
|
|
179
175
|
end
|
|
@@ -181,8 +177,7 @@ module Upkeep
|
|
|
181
177
|
def instrument_delivery_enqueue_error(changes, error)
|
|
182
178
|
ActiveSupport::Notifications.instrument(
|
|
183
179
|
DELIVERY_ENQUEUE_ERROR,
|
|
184
|
-
|
|
185
|
-
queue: configuration.delivery_queue,
|
|
180
|
+
inline: configuration.deliver_inline,
|
|
186
181
|
change_count: changes.size,
|
|
187
182
|
error_class: error.class.name,
|
|
188
183
|
error_message: error.message
|
|
@@ -354,6 +349,52 @@ module Upkeep
|
|
|
354
349
|
"config.upkeep.subscription_store = :memory in development/test."
|
|
355
350
|
end
|
|
356
351
|
|
|
352
|
+
def validate_cluster_safety!(environment:, guard: nil)
|
|
353
|
+
guard ||= ClusterGuard.new(
|
|
354
|
+
cable_adapter: resolved_action_cable_adapter,
|
|
355
|
+
worker_count: detected_worker_count,
|
|
356
|
+
subscription_store: configuration.subscription_store,
|
|
357
|
+
environment: environment
|
|
358
|
+
)
|
|
359
|
+
return true if guard.problems.empty?
|
|
360
|
+
raise ConfigurationError, guard.message if guard.error?
|
|
361
|
+
|
|
362
|
+
log_cluster_warning(guard.message)
|
|
363
|
+
true
|
|
364
|
+
end
|
|
365
|
+
|
|
366
|
+
def log_cluster_warning(message)
|
|
367
|
+
return if @cluster_warning_logged
|
|
368
|
+
|
|
369
|
+
@cluster_warning_logged = true
|
|
370
|
+
if defined?(::Rails) && ::Rails.respond_to?(:logger) && ::Rails.logger
|
|
371
|
+
::Rails.logger.warn("[upkeep] #{message}")
|
|
372
|
+
else
|
|
373
|
+
warn("[upkeep] #{message}")
|
|
374
|
+
end
|
|
375
|
+
end
|
|
376
|
+
|
|
377
|
+
def resolved_action_cable_adapter
|
|
378
|
+
return unless defined?(::ActionCable) && ::ActionCable.respond_to?(:server)
|
|
379
|
+
|
|
380
|
+
cable = ::ActionCable.server.config.cable
|
|
381
|
+
cable && (cable["adapter"] || cable[:adapter])
|
|
382
|
+
rescue StandardError
|
|
383
|
+
nil
|
|
384
|
+
end
|
|
385
|
+
|
|
386
|
+
def detected_worker_count
|
|
387
|
+
puma_worker_count || ENV["WEB_CONCURRENCY"].to_i
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
def puma_worker_count
|
|
391
|
+
return unless defined?(::Puma) && ::Puma.respond_to?(:cli_config)
|
|
392
|
+
|
|
393
|
+
::Puma.cli_config&.options&.[](:workers)
|
|
394
|
+
rescue StandardError
|
|
395
|
+
nil
|
|
396
|
+
end
|
|
397
|
+
|
|
357
398
|
def production_environment?(environment)
|
|
358
399
|
environment.to_s == "production"
|
|
359
400
|
end
|
data/lib/upkeep/runtime.rb
CHANGED
|
@@ -783,11 +783,13 @@ module Upkeep
|
|
|
783
783
|
value = super
|
|
784
784
|
return value unless Observation.recording?
|
|
785
785
|
|
|
786
|
+
id = primary_key_value(attr_name, value)
|
|
786
787
|
dependency = Dependencies::ActiveRecordAttribute.new(
|
|
787
788
|
table: self.class.table_name,
|
|
788
789
|
model: self.class.name,
|
|
789
|
-
id:
|
|
790
|
-
attribute: attr_name.to_s
|
|
790
|
+
id: id,
|
|
791
|
+
attribute: attr_name.to_s,
|
|
792
|
+
scope: id.nil? ? upkeep_fresh_record_scope : nil
|
|
791
793
|
)
|
|
792
794
|
|
|
793
795
|
Observation.record_dependency(dependency)
|
|
@@ -806,6 +808,41 @@ module Upkeep
|
|
|
806
808
|
rescue StandardError
|
|
807
809
|
nil
|
|
808
810
|
end
|
|
811
|
+
|
|
812
|
+
def upkeep_fresh_record_scope
|
|
813
|
+
return nil unless new_record?
|
|
814
|
+
|
|
815
|
+
self.class.reflect_on_all_associations(:belongs_to).each_with_object({}) do |reflection, scope|
|
|
816
|
+
foreign_key = reflection.foreign_key
|
|
817
|
+
next if foreign_key.is_a?(Array)
|
|
818
|
+
|
|
819
|
+
foreign_key = foreign_key.to_s
|
|
820
|
+
fk_value = upkeep_scope_attribute_value(foreign_key)
|
|
821
|
+
next if fk_value.nil?
|
|
822
|
+
|
|
823
|
+
if reflection.polymorphic?
|
|
824
|
+
foreign_type = reflection.foreign_type.to_s
|
|
825
|
+
type_value = upkeep_scope_attribute_value(foreign_type)
|
|
826
|
+
next if type_value.nil?
|
|
827
|
+
|
|
828
|
+
scope[foreign_type] = type_value
|
|
829
|
+
end
|
|
830
|
+
|
|
831
|
+
scope[foreign_key] = fk_value
|
|
832
|
+
end
|
|
833
|
+
rescue StandardError
|
|
834
|
+
nil
|
|
835
|
+
end
|
|
836
|
+
|
|
837
|
+
def upkeep_scope_attribute_value(name)
|
|
838
|
+
return nil unless @attributes.key?(name)
|
|
839
|
+
|
|
840
|
+
value = @attributes.fetch_value(name)
|
|
841
|
+
case value
|
|
842
|
+
when true, false, Numeric, String, Symbol
|
|
843
|
+
value
|
|
844
|
+
end
|
|
845
|
+
end
|
|
809
846
|
end
|
|
810
847
|
|
|
811
848
|
# Hooks `cache_key_with_version` so that any caller (Rails fragment caching,
|
|
@@ -8,8 +8,8 @@ module Upkeep
|
|
|
8
8
|
|
|
9
9
|
module_function
|
|
10
10
|
|
|
11
|
-
def stream_name(target:, identity_signature:, sharing_signature:)
|
|
12
|
-
digest = Digest::SHA256.hexdigest([target.kind, target.id, identity_signature, sharing_signature].inspect)[0, 32]
|
|
11
|
+
def stream_name(target:, identity_signature:, sharing_signature:, deployment_signature:)
|
|
12
|
+
digest = Digest::SHA256.hexdigest([target.kind, target.id, identity_signature, sharing_signature, deployment_signature].inspect)[0, 32]
|
|
13
13
|
"#{PREFIX}:#{digest}"
|
|
14
14
|
end
|
|
15
15
|
|
|
@@ -47,7 +47,8 @@ module Upkeep
|
|
|
47
47
|
stream_name(
|
|
48
48
|
target: target,
|
|
49
49
|
identity_signature: identity_signature,
|
|
50
|
-
sharing_signature: signature_for(recipe)
|
|
50
|
+
sharing_signature: signature_for(recipe),
|
|
51
|
+
deployment_signature: deployment_signature_for(graph, frame.id)
|
|
51
52
|
)
|
|
52
53
|
end.uniq.sort
|
|
53
54
|
end
|
|
@@ -62,6 +63,19 @@ module Upkeep
|
|
|
62
63
|
Digest::SHA256.hexdigest(identity_dependencies.map(&:identity_key).sort_by(&:inspect).inspect)[0, 16]
|
|
63
64
|
end
|
|
64
65
|
|
|
66
|
+
# Deployment-stable request reads (host, protocol, ...) do not partition viewers, but
|
|
67
|
+
# their fingerprinted values must still scope the stream: folding them into the name
|
|
68
|
+
# makes cross-host sharing impossible by construction.
|
|
69
|
+
def deployment_signature_for(graph, frame_id)
|
|
70
|
+
deployment_dependencies = graph.contained_node_ids(frame_id)
|
|
71
|
+
.flat_map { |owner_id| graph.dependencies_for(owner_id) }
|
|
72
|
+
.select { |dependency| Dependencies.deployment_stable_request?(dependency) }
|
|
73
|
+
.uniq(&:cache_key)
|
|
74
|
+
return "none" if deployment_dependencies.empty?
|
|
75
|
+
|
|
76
|
+
Digest::SHA256.hexdigest(deployment_dependencies.map(&:identity_key).sort_by(&:inspect).inspect)[0, 16]
|
|
77
|
+
end
|
|
78
|
+
|
|
65
79
|
def target_for_frame(frame)
|
|
66
80
|
case frame.payload.fetch(:kind)
|
|
67
81
|
when "render_site"
|
|
@@ -5,7 +5,7 @@ require "active_support/notifications"
|
|
|
5
5
|
require "securerandom"
|
|
6
6
|
require_relative "active_record_subscription_persistence"
|
|
7
7
|
require_relative "active_registry"
|
|
8
|
-
require_relative "
|
|
8
|
+
require_relative "base_store"
|
|
9
9
|
require_relative "json_snapshot"
|
|
10
10
|
require_relative "layered_reverse_index"
|
|
11
11
|
require_relative "persistent_reverse_index"
|
|
@@ -14,12 +14,12 @@ require_relative "store"
|
|
|
14
14
|
|
|
15
15
|
module Upkeep
|
|
16
16
|
module Subscriptions
|
|
17
|
-
class ActiveRecordStore
|
|
17
|
+
class ActiveRecordStore < BaseStore
|
|
18
18
|
LOOKUP_NOTIFICATION = LayeredReverseIndex::LOOKUP_NOTIFICATION
|
|
19
19
|
REGISTER_NOTIFICATION = "register_subscription_store.upkeep"
|
|
20
20
|
ACTIVATE_NOTIFICATION = "activate_subscription_store.upkeep"
|
|
21
21
|
PERSIST_NOTIFICATION = ActiveRecordSubscriptionPersistence::PERSIST_NOTIFICATION
|
|
22
|
-
DURABILITY_MODE = "
|
|
22
|
+
DURABILITY_MODE = "sync_subscription_row_index_on_subscribe"
|
|
23
23
|
INDEX_DURABILITY = "on_subscribe"
|
|
24
24
|
REQUIRED_SCHEMA = {
|
|
25
25
|
"upkeep_subscriptions" => {
|
|
@@ -115,7 +115,6 @@ module Upkeep
|
|
|
115
115
|
store: "active_record",
|
|
116
116
|
pending_index: pending_registry
|
|
117
117
|
)
|
|
118
|
-
@durable_writer = AsyncDurableWriter.new { |jobs| persistence.persist_jobs(jobs) }
|
|
119
118
|
end
|
|
120
119
|
|
|
121
120
|
def self.available?(connect: false)
|
|
@@ -170,71 +169,29 @@ module Upkeep
|
|
|
170
169
|
private_class_method :expected_column_description
|
|
171
170
|
|
|
172
171
|
def register(subscriber_id:, recorder:, metadata: {}, entries: nil)
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
ActiveSupport::Notifications.instrument(REGISTER_NOTIFICATION, payload) do
|
|
176
|
-
register_subscription(subscriber_id, recorder, metadata, entries: entries, payload: payload)
|
|
177
|
-
end
|
|
178
|
-
else
|
|
179
|
-
register_subscription(subscriber_id, recorder, metadata, entries: entries)
|
|
172
|
+
subscription = with_optional_notification(REGISTER_NOTIFICATION, { store: "active_record" }) do |payload|
|
|
173
|
+
register_subscription(subscriber_id, recorder, metadata, entries: entries, payload: payload)
|
|
180
174
|
end
|
|
175
|
+
trim_opportunistically
|
|
176
|
+
subscription
|
|
181
177
|
end
|
|
182
178
|
|
|
183
|
-
def drain = durable_writer.drain
|
|
184
|
-
|
|
185
179
|
def shutdown
|
|
186
180
|
clear_deferred_index_writes
|
|
187
|
-
durable_writer.shutdown
|
|
188
181
|
end
|
|
189
182
|
|
|
190
183
|
def activate(id)
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
ActiveSupport::Notifications.instrument(ACTIVATE_NOTIFICATION, payload) do
|
|
194
|
-
activate_subscription(id, payload: payload)
|
|
195
|
-
end
|
|
196
|
-
else
|
|
197
|
-
activate_subscription(id)
|
|
184
|
+
with_optional_notification(ACTIVATE_NOTIFICATION, { store: "active_record", subscription_id: id }) do |payload|
|
|
185
|
+
activate_subscription(id, payload: payload)
|
|
198
186
|
end
|
|
199
187
|
end
|
|
200
188
|
|
|
201
|
-
def
|
|
202
|
-
|
|
203
|
-
metadata = { "last_seen_at" => now.utc.iso8601 }
|
|
204
|
-
pending_registry.touch(id, metadata: metadata)
|
|
205
|
-
active_registry.touch(id, metadata: metadata)
|
|
206
|
-
activate(id)
|
|
207
|
-
durable_writer.drain
|
|
208
|
-
persistence.touch(id, metadata: metadata, now: now)
|
|
209
|
-
end
|
|
210
|
-
|
|
211
|
-
def unregister(ids)
|
|
212
|
-
ids = Array(ids)
|
|
213
|
-
pending_registry.unregister(ids)
|
|
214
|
-
active_registry.unregister(ids)
|
|
215
|
-
delete_deferred_index_writes(ids)
|
|
216
|
-
persisted_ids = durable_writer.cancel(ids)
|
|
217
|
-
persistence.delete(persisted_ids)
|
|
218
|
-
ids.size
|
|
219
|
-
end
|
|
220
|
-
|
|
221
|
-
def prune_stale!(older_than:)
|
|
222
|
-
durable_writer.drain
|
|
223
|
-
stale_ids = persistence.prune_stale!(older_than: older_than)
|
|
189
|
+
def prune_stale!(older_than: stale_threshold, limit: nil)
|
|
190
|
+
stale_ids = persistence.prune_stale!(older_than: older_than, limit: limit)
|
|
224
191
|
active_registry.unregister(stale_ids)
|
|
225
192
|
stale_ids.size
|
|
226
193
|
end
|
|
227
194
|
|
|
228
|
-
def fetch(id)
|
|
229
|
-
active_registry.fetch(id) || pending_registry.fetch(id) || persistence.fetch(id)
|
|
230
|
-
rescue ActiveRecord::RecordNotFound
|
|
231
|
-
raise NotFound, id
|
|
232
|
-
end
|
|
233
|
-
|
|
234
|
-
def explain(id)
|
|
235
|
-
fetch(id).explain
|
|
236
|
-
end
|
|
237
|
-
|
|
238
195
|
def subscriptions
|
|
239
196
|
persistent_count = persistence.count
|
|
240
197
|
in_memory_subscriptions = (active_registry.subscriptions + pending_registry.subscriptions).to_h do |subscription|
|
|
@@ -252,7 +209,6 @@ module Upkeep
|
|
|
252
209
|
|
|
253
210
|
def reset
|
|
254
211
|
clear_deferred_index_writes
|
|
255
|
-
durable_writer.drain
|
|
256
212
|
pending_registry.reset
|
|
257
213
|
active_registry.reset
|
|
258
214
|
persistence.reset
|
|
@@ -274,7 +230,27 @@ module Upkeep
|
|
|
274
230
|
|
|
275
231
|
private
|
|
276
232
|
|
|
277
|
-
attr_reader :subscription_record, :index_record, :shape_index_record, :index_builder, :pending_registry, :active_registry, :persistence
|
|
233
|
+
attr_reader :subscription_record, :index_record, :shape_index_record, :index_builder, :pending_registry, :active_registry, :persistence
|
|
234
|
+
|
|
235
|
+
def store_label
|
|
236
|
+
"active_record"
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
def after_touch(id, metadata:, now:)
|
|
240
|
+
activate(id)
|
|
241
|
+
persistence.touch(id, metadata: metadata, now: now)
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
def after_unregister(ids)
|
|
245
|
+
delete_deferred_index_writes(ids)
|
|
246
|
+
persistence.delete(ids)
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
def fetch_missing(id)
|
|
250
|
+
persistence.fetch(id)
|
|
251
|
+
rescue ActiveRecord::RecordNotFound
|
|
252
|
+
raise NotFound, id
|
|
253
|
+
end
|
|
278
254
|
|
|
279
255
|
def register_subscription(subscriber_id, recorder, metadata, entries: nil, payload: nil)
|
|
280
256
|
recorder.flush_pending_dependencies if recorder.respond_to?(:flush_pending_dependencies)
|
|
@@ -296,7 +272,7 @@ module Upkeep
|
|
|
296
272
|
end
|
|
297
273
|
|
|
298
274
|
pending_registry.register(subscription, entries: entries)
|
|
299
|
-
|
|
275
|
+
persist_now(subscription, entries, operation: :persist_subscription)
|
|
300
276
|
defer_index_write(subscription, entries)
|
|
301
277
|
subscription
|
|
302
278
|
end
|
|
@@ -312,7 +288,7 @@ module Upkeep
|
|
|
312
288
|
unless source == :active
|
|
313
289
|
active_registry.register(subscription, entries: entries)
|
|
314
290
|
pending_registry.unregister(id)
|
|
315
|
-
|
|
291
|
+
persist_now(subscription, entries, operation: :persist_index)
|
|
316
292
|
end
|
|
317
293
|
|
|
318
294
|
if payload
|
|
@@ -326,6 +302,25 @@ module Upkeep
|
|
|
326
302
|
true
|
|
327
303
|
end
|
|
328
304
|
|
|
305
|
+
# Synchronous so other processes see the row (registration) and the index
|
|
306
|
+
# entries (activation) immediately. Persistence failures degrade to
|
|
307
|
+
# in-process liveness instead of failing the request or the activation.
|
|
308
|
+
def persist_now(subscription, entries, operation:)
|
|
309
|
+
persistence.persist_jobs([ActiveRecordSubscriptionPersistence::Job.new(subscription, entries, operation)])
|
|
310
|
+
rescue StandardError => error
|
|
311
|
+
warn_persist_failure(subscription, operation, error)
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
def warn_persist_failure(subscription, operation, error)
|
|
315
|
+
return unless defined?(::Rails) && ::Rails.respond_to?(:logger) && ::Rails.logger
|
|
316
|
+
|
|
317
|
+
target = operation == :persist_subscription ? "subscription row" : "index entries"
|
|
318
|
+
::Rails.logger.warn(
|
|
319
|
+
"Upkeep could not persist #{target} for #{subscription.id} " \
|
|
320
|
+
"(#{error.class}: #{error.message}); the subscription stays live in this process only"
|
|
321
|
+
)
|
|
322
|
+
end
|
|
323
|
+
|
|
329
324
|
def defer_index_write(subscription, entries)
|
|
330
325
|
@deferred_index_mutex.synchronize do
|
|
331
326
|
@deferred_index_writes[subscription.id] = DeferredIndexWrite.new(subscription, entries)
|
|
@@ -378,10 +373,6 @@ module Upkeep
|
|
|
378
373
|
index_builder.entries_for_subscription(subscription)
|
|
379
374
|
end
|
|
380
375
|
end
|
|
381
|
-
|
|
382
|
-
def next_subscription_id
|
|
383
|
-
"subscription-#{SecureRandom.uuid}"
|
|
384
|
-
end
|
|
385
376
|
end
|
|
386
377
|
end
|
|
387
378
|
end
|
|
@@ -12,6 +12,7 @@ module Upkeep
|
|
|
12
12
|
class ActiveRecordSubscriptionPersistence
|
|
13
13
|
PERSIST_NOTIFICATION = "persist_subscription_store.upkeep"
|
|
14
14
|
INDEX_ENTRIES_SNAPSHOT_KEY = "__upkeep_index_entries"
|
|
15
|
+
Job = Data.define(:subscription, :entries, :operation)
|
|
15
16
|
|
|
16
17
|
def initialize(subscription_record:, index_record:, shape_index_record:, index_builder:)
|
|
17
18
|
@subscription_record = subscription_record
|
|
@@ -54,8 +55,10 @@ module Upkeep
|
|
|
54
55
|
end
|
|
55
56
|
end
|
|
56
57
|
|
|
57
|
-
def prune_stale!(older_than:)
|
|
58
|
-
|
|
58
|
+
def prune_stale!(older_than:, limit: nil)
|
|
59
|
+
stale = subscription_record.where(subscription_record.arel_table[:updated_at].lt(older_than))
|
|
60
|
+
stale = stale.order(:updated_at).limit(limit) if limit
|
|
61
|
+
stale_ids = stale.pluck(:id)
|
|
59
62
|
return [] if stale_ids.empty?
|
|
60
63
|
|
|
61
64
|
delete(stale_ids)
|
|
@@ -254,7 +257,7 @@ module Upkeep
|
|
|
254
257
|
lookup_attribute: attribute.to_s,
|
|
255
258
|
dependency_table: dependency_key.fetch(:table).to_s,
|
|
256
259
|
dependency_predicate_digest: nil,
|
|
257
|
-
dependency_metadata_snapshot: nil
|
|
260
|
+
dependency_metadata_snapshot: dependency_key[:scope] ? dump(scope: dependency_key[:scope]) : nil
|
|
258
261
|
}
|
|
259
262
|
when :active_record_collection_column
|
|
260
263
|
_type, table, attribute = lookup_key
|
|
@@ -75,13 +75,6 @@ module Upkeep
|
|
|
75
75
|
@reverse_index.summary.merge(subscriptions: @subscriptions.size)
|
|
76
76
|
end
|
|
77
77
|
end
|
|
78
|
-
|
|
79
|
-
private
|
|
80
|
-
|
|
81
|
-
def rebuild_reverse_index!
|
|
82
|
-
@reverse_index = ReverseIndex.new
|
|
83
|
-
@subscriptions.each_value { |subscription| @reverse_index.index(subscription) }
|
|
84
|
-
end
|
|
85
78
|
end
|
|
86
79
|
end
|
|
87
80
|
end
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "securerandom"
|
|
4
|
+
require "active_support/notifications"
|
|
5
|
+
|
|
6
|
+
module Upkeep
|
|
7
|
+
module Subscriptions
|
|
8
|
+
# Shared pending -> active registry choreography for the in-memory and
|
|
9
|
+
# Active Record stores. Subclasses hold the @pending_registry / @active_registry
|
|
10
|
+
# instances and supply their store-specific tails through the documented hooks.
|
|
11
|
+
class BaseStore
|
|
12
|
+
# Opportunistic trim (Solid Cache/Solid Cable style): every TRIM_EVERY
|
|
13
|
+
# registrations the store deletes at most TRIM_BATCH_LIMIT subscriptions
|
|
14
|
+
# older than the configured subscription TTL. A deterministic counter is
|
|
15
|
+
# used instead of rand so the cadence is reproducible in tests.
|
|
16
|
+
TRIM_EVERY = 100
|
|
17
|
+
TRIM_BATCH_LIMIT = 500
|
|
18
|
+
PRUNE_NOTIFICATION = "prune.upkeep"
|
|
19
|
+
DEFAULT_SUBSCRIPTION_TTL = 24 * 60 * 60
|
|
20
|
+
|
|
21
|
+
def touch(id, now: Time.now)
|
|
22
|
+
fetch(id)
|
|
23
|
+
metadata = { "last_seen_at" => now.utc.iso8601 }
|
|
24
|
+
pending_registry.touch(id, metadata: metadata)
|
|
25
|
+
active_registry.touch(id, metadata: metadata)
|
|
26
|
+
after_touch(id, metadata: metadata, now: now)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def unregister(ids)
|
|
30
|
+
ids = Array(ids)
|
|
31
|
+
before_unregister(ids)
|
|
32
|
+
pending_registry.unregister(ids)
|
|
33
|
+
active_registry.unregister(ids)
|
|
34
|
+
after_unregister(ids)
|
|
35
|
+
ids.size
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def fetch(id)
|
|
39
|
+
active_registry.fetch(id) || pending_registry.fetch(id) || fetch_missing(id)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def explain(id)
|
|
43
|
+
fetch(id).explain
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
# Wrap a unit of work in an optional ActiveSupport notification: when a
|
|
49
|
+
# listener is attached the block runs inside #instrument with the given
|
|
50
|
+
# payload, otherwise it runs with a nil payload and no instrumentation.
|
|
51
|
+
def with_optional_notification(notification, payload)
|
|
52
|
+
if ActiveSupport::Notifications.notifier.listening?(notification)
|
|
53
|
+
ActiveSupport::Notifications.instrument(notification, payload) do
|
|
54
|
+
yield payload
|
|
55
|
+
end
|
|
56
|
+
else
|
|
57
|
+
yield nil
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Runs one bounded prune batch every TRIM_EVERY registrations. Failures
|
|
62
|
+
# never propagate into the registration path: they are logged and the
|
|
63
|
+
# stale rows stay until the next trim.
|
|
64
|
+
def trim_opportunistically
|
|
65
|
+
@trim_registration_count = (@trim_registration_count || 0) + 1
|
|
66
|
+
return unless (@trim_registration_count % TRIM_EVERY).zero?
|
|
67
|
+
|
|
68
|
+
payload = { store: store_label, limit: TRIM_BATCH_LIMIT }
|
|
69
|
+
ActiveSupport::Notifications.instrument(PRUNE_NOTIFICATION, payload) do
|
|
70
|
+
payload[:pruned] = prune_stale!(limit: TRIM_BATCH_LIMIT)
|
|
71
|
+
end
|
|
72
|
+
rescue StandardError => error
|
|
73
|
+
warn_trim_failure(error)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def warn_trim_failure(error)
|
|
77
|
+
return unless defined?(::Rails) && ::Rails.respond_to?(:logger) && ::Rails.logger
|
|
78
|
+
|
|
79
|
+
::Rails.logger.warn(
|
|
80
|
+
"Upkeep opportunistic subscription prune failed " \
|
|
81
|
+
"(#{error.class}: #{error.message}); stale subscriptions stay until the next trim"
|
|
82
|
+
)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def stale_threshold
|
|
86
|
+
Time.now - subscription_ttl
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def subscription_ttl
|
|
90
|
+
if defined?(Upkeep::Rails) && Upkeep::Rails.respond_to?(:configuration)
|
|
91
|
+
Upkeep::Rails.configuration.subscription_ttl
|
|
92
|
+
else
|
|
93
|
+
DEFAULT_SUBSCRIPTION_TTL
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def before_unregister(ids); end
|
|
98
|
+
|
|
99
|
+
def after_unregister(ids); end
|
|
100
|
+
|
|
101
|
+
def next_subscription_id
|
|
102
|
+
"subscription-#{SecureRandom.uuid}"
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "active_support/notifications"
|
|
4
|
+
require_relative "lookup_instrumentation"
|
|
4
5
|
|
|
5
6
|
module Upkeep
|
|
6
7
|
module Subscriptions
|
|
7
8
|
class LayeredReverseIndex
|
|
8
|
-
|
|
9
|
+
include LookupInstrumentation
|
|
10
|
+
|
|
11
|
+
LOOKUP_NOTIFICATION = LookupInstrumentation::LOOKUP_NOTIFICATION
|
|
9
12
|
|
|
10
13
|
def initialize(active_index:, persistent_index:, persistent_count:, store:, pending_index: nil)
|
|
11
14
|
@active_index = active_index
|
|
@@ -15,17 +18,6 @@ module Upkeep
|
|
|
15
18
|
@pending_index = pending_index
|
|
16
19
|
end
|
|
17
20
|
|
|
18
|
-
def entries_for(changes)
|
|
19
|
-
if ActiveSupport::Notifications.notifier.listening?(LOOKUP_NOTIFICATION)
|
|
20
|
-
payload = { changes: Array(changes).size, store: store }
|
|
21
|
-
ActiveSupport::Notifications.instrument(LOOKUP_NOTIFICATION, payload) do
|
|
22
|
-
entries_for_with_payload(changes, payload)
|
|
23
|
-
end
|
|
24
|
-
else
|
|
25
|
-
entries_for_without_payload(changes)
|
|
26
|
-
end
|
|
27
|
-
end
|
|
28
|
-
|
|
29
21
|
def entries_for_without_payload(changes)
|
|
30
22
|
active_entries = active_index.entries_for(changes)
|
|
31
23
|
return persistent_index.entries_for(changes) if active_index.count.zero?
|
|
@@ -101,6 +93,10 @@ module Upkeep
|
|
|
101
93
|
|
|
102
94
|
attr_reader :active_index, :persistent_index, :persistent_count, :store, :pending_index
|
|
103
95
|
|
|
96
|
+
def lookup_store
|
|
97
|
+
store
|
|
98
|
+
end
|
|
99
|
+
|
|
104
100
|
def persistent_subscription_count
|
|
105
101
|
persistent_count.call
|
|
106
102
|
end
|
|
@@ -122,7 +118,7 @@ module Upkeep
|
|
|
122
118
|
def apply_miss_reason(payload, active_entries:, persistent_entries:, pending_entries:)
|
|
123
119
|
return if active_entries.any? || persistent_entries.any?
|
|
124
120
|
|
|
125
|
-
payload[:miss_reason] = pending_entries
|
|
121
|
+
payload[:miss_reason] = miss_reason(pending_entries)
|
|
126
122
|
end
|
|
127
123
|
end
|
|
128
124
|
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_support/notifications"
|
|
4
|
+
|
|
5
|
+
module Upkeep
|
|
6
|
+
module Subscriptions
|
|
7
|
+
# Shared lookup instrumentation for the memory and layered reverse indexes:
|
|
8
|
+
# one notification name, one entries_for dispatch, one miss-reason rule.
|
|
9
|
+
# Including classes provide #lookup_store, #entries_for_with_payload and
|
|
10
|
+
# #entries_for_without_payload.
|
|
11
|
+
module LookupInstrumentation
|
|
12
|
+
LOOKUP_NOTIFICATION = "lookup_subscription_index.upkeep"
|
|
13
|
+
|
|
14
|
+
def entries_for(changes)
|
|
15
|
+
if ActiveSupport::Notifications.notifier.listening?(LOOKUP_NOTIFICATION)
|
|
16
|
+
payload = { changes: Array(changes).size, store: lookup_store }
|
|
17
|
+
ActiveSupport::Notifications.instrument(LOOKUP_NOTIFICATION, payload) do
|
|
18
|
+
entries_for_with_payload(changes, payload)
|
|
19
|
+
end
|
|
20
|
+
else
|
|
21
|
+
entries_for_without_payload(changes)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
private
|
|
26
|
+
|
|
27
|
+
def miss_reason(pending_entries)
|
|
28
|
+
pending_entries.any? ? "not_activated_yet" : "no_matching_subscriber"
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -188,10 +188,13 @@ module Upkeep
|
|
|
188
188
|
source = attributes.fetch(:dependency_source).to_sym
|
|
189
189
|
case source
|
|
190
190
|
when :active_record_attribute
|
|
191
|
+
metadata_snapshot = attributes[:dependency_metadata_snapshot]
|
|
192
|
+
metadata = metadata_snapshot && JsonSnapshot.load(metadata_snapshot)
|
|
191
193
|
Dependencies::ActiveRecordAttribute.new(
|
|
192
194
|
table: attributes.fetch(:dependency_table),
|
|
193
195
|
id: attributes[:lookup_record_id_snapshot] && JsonSnapshot.load(attributes.fetch(:lookup_record_id_snapshot)),
|
|
194
|
-
attribute: attributes.fetch(:lookup_attribute)
|
|
196
|
+
attribute: attributes.fetch(:lookup_attribute),
|
|
197
|
+
scope: metadata && metadata[:scope]
|
|
195
198
|
)
|
|
196
199
|
when :active_record_collection, :active_record_query
|
|
197
200
|
metadata = JsonSnapshot.load(attributes.fetch(:dependency_metadata_snapshot))
|