active_storage_quota 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +52 -0
  3. data/LICENSE.txt +21 -0
  4. data/README.md +404 -0
  5. data/app/controllers/active_storage_quota/direct_uploads_controller.rb +96 -0
  6. data/db/migrate/20260917000001_create_active_storage_quota_tables.rb +72 -0
  7. data/db/migrate/20260918000001_create_active_storage_quota_charges.rb +45 -0
  8. data/lib/active_storage_quota/account.rb +200 -0
  9. data/lib/active_storage_quota/attachable.rb +44 -0
  10. data/lib/active_storage_quota/attachment_accounting.rb +144 -0
  11. data/lib/active_storage_quota/audit/counter_audit.rb +62 -0
  12. data/lib/active_storage_quota/audit/finding.rb +17 -0
  13. data/lib/active_storage_quota/audit/ledger_audit.rb +164 -0
  14. data/lib/active_storage_quota/audit/result.rb +106 -0
  15. data/lib/active_storage_quota/audit.rb +75 -0
  16. data/lib/active_storage_quota/backfill/cursor.rb +64 -0
  17. data/lib/active_storage_quota/backfill/result.rb +146 -0
  18. data/lib/active_storage_quota/backfill.rb +310 -0
  19. data/lib/active_storage_quota/blob_accounting.rb +44 -0
  20. data/lib/active_storage_quota/charge.rb +379 -0
  21. data/lib/active_storage_quota/configuration.rb +37 -0
  22. data/lib/active_storage_quota/definition.rb +100 -0
  23. data/lib/active_storage_quota/engine.rb +28 -0
  24. data/lib/active_storage_quota/errors.rb +115 -0
  25. data/lib/active_storage_quota/owner.rb +154 -0
  26. data/lib/active_storage_quota/owner_resolver.rb +402 -0
  27. data/lib/active_storage_quota/owner_source.rb +88 -0
  28. data/lib/active_storage_quota/reconciliation/budget.rb +70 -0
  29. data/lib/active_storage_quota/reconciliation/counter_reconciler.rb +78 -0
  30. data/lib/active_storage_quota/reconciliation/ledger_reconciler.rb +250 -0
  31. data/lib/active_storage_quota/reconciliation/result.rb +107 -0
  32. data/lib/active_storage_quota/reconciliation.rb +72 -0
  33. data/lib/active_storage_quota/record.rb +20 -0
  34. data/lib/active_storage_quota/reservation.rb +297 -0
  35. data/lib/active_storage_quota/version.rb +5 -0
  36. data/lib/active_storage_quota.rb +207 -0
  37. data/lib/tasks/active_storage_quota.rake +11 -0
  38. data/lib/tasks/active_storage_quota_audit.rake +15 -0
  39. data/lib/tasks/active_storage_quota_backfill.rake +104 -0
  40. data/lib/tasks/active_storage_quota_reconcile.rake +100 -0
  41. metadata +150 -0
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ class CreateActiveStorageQuotaCharges < ActiveRecord::Migration[7.1]
4
+ def change
5
+ create_table :active_storage_quota_charges do |t|
6
+ t.references :quota_account, null: false, index: false,
7
+ foreign_key: { to_table: :active_storage_quota_accounts }
8
+
9
+ # blob_id is a string for the same reason owner_id is: Active Storage's
10
+ # primary key type follows the host application's generator config, so it
11
+ # is bigint in one application and uuid in the next. A string also lets
12
+ # the Active Storage tables live in a different database, which a foreign
13
+ # key would forbid -- and there is deliberately no foreign key here.
14
+ t.string :blob_id, null: false, limit: 255
15
+
16
+ # Kept rather than joined from the blob: the blob may already be gone when
17
+ # we need to decrement, and the Active Storage tables may not even be
18
+ # reachable from this connection.
19
+ t.bigint :byte_size, null: false
20
+
21
+ # How many attachments of this account reference this blob. One charge
22
+ # covers all of them; the account is charged the blob's size once.
23
+ t.integer :attachment_count, null: false, default: 0
24
+
25
+ t.timestamps
26
+
27
+ t.index %i[quota_account_id blob_id], unique: true,
28
+ name: "index_asq_charges_on_account_and_blob"
29
+ # The blob-destroy path fans out across every account holding a charge.
30
+ t.index :blob_id, name: "index_asq_charges_on_blob_id"
31
+
32
+ t.check_constraint "byte_size >= 0",
33
+ name: "asq_charges_byte_size_non_negative"
34
+ t.check_constraint "attachment_count >= 0",
35
+ name: "asq_charges_attachment_count_non_negative"
36
+ end
37
+
38
+ # Correlates a direct-upload hold with the blob it was taken for. Nullable
39
+ # because server-side attachments never need a hold.
40
+ add_column :active_storage_quota_reservations, :blob_id, :string, limit: 255
41
+ add_index :active_storage_quota_reservations,
42
+ %i[blob_id quota_account_id status],
43
+ name: "index_asq_reservations_on_blob_and_account"
44
+ end
45
+ end
@@ -0,0 +1,200 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveStorageQuota
4
+ # The byte counters for one quota owner and scope.
5
+ #
6
+ # +used_bytes+ is committed storage; +reserved_bytes+ is storage claimed by
7
+ # in-flight uploads. Both are maintained by the reservation system rather
8
+ # than by this model, which is why there are no callbacks here.
9
+ #
10
+ # The validations mirror the database CHECK constraints so that mistakes
11
+ # surface as validation errors in application code. The constraints remain
12
+ # authoritative: quota counters are updated by conditional UPDATE statements
13
+ # that bypass validations by design.
14
+ class Account < Record
15
+ self.table_name = "active_storage_quota_accounts"
16
+
17
+ # optional: false rather than relying on belongs_to_required_by_default,
18
+ # which depends on the host application's Rails defaults.
19
+ belongs_to :owner, polymorphic: true, optional: false
20
+
21
+ has_many :reservations,
22
+ class_name: "ActiveStorageQuota::Reservation",
23
+ foreign_key: :quota_account_id,
24
+ inverse_of: :quota_account,
25
+ dependent: :delete_all
26
+
27
+ has_many :charges,
28
+ class_name: "ActiveStorageQuota::Charge",
29
+ foreign_key: :quota_account_id,
30
+ inverse_of: :quota_account,
31
+ dependent: :delete_all
32
+
33
+ validates :scope_name, presence: true
34
+ validates :used_bytes, :reserved_bytes,
35
+ numericality: { only_integer: true, greater_than_or_equal_to: 0 }
36
+
37
+ # No uniqueness validation on [owner, scope_name]. It would add a SELECT to
38
+ # every save and still lose the race it appears to prevent; the unique index
39
+ # is what actually enforces it.
40
+
41
+ class << self
42
+ # Claims +byte_size+ bytes for +owner+ and returns the resulting
43
+ # Reservation. Raises QuotaExceeded when the bytes do not fit under
44
+ # +limit_bytes+, and ArgumentError for a byte size that is not an exact
45
+ # non-negative Integer.
46
+ #
47
+ # +limit_bytes+ is the limit already resolved for this call. Passing it in
48
+ # rather than resolving it here is what makes the guarantee precise: the
49
+ # reservation is admitted against that value, atomically. A concurrent
50
+ # change to whatever produced it is not serialised against -- see the
51
+ # README on dynamic limits.
52
+ #
53
+ # Everything happens in one transaction. There is never an incremented
54
+ # reserved_bytes without its reservation row, nor the reverse.
55
+ def reserve!(owner:, byte_size:, limit_bytes:, blob_id: nil,
56
+ scope_name: ActiveStorageQuota::DEFAULT_SCOPE)
57
+ byte_size = validate_byte_size!(byte_size)
58
+ now = Time.current
59
+
60
+ # requires_new so the whole claim is one atomic unit even inside a host
61
+ # application transaction: if the reservation INSERT fails, the capacity
62
+ # claim rolls back with it whether or not the caller rescues.
63
+ transaction(requires_new: true) do
64
+ account = find_or_create_for!(owner, scope_name)
65
+ account.claim_capacity!(byte_size, limit_bytes, now)
66
+
67
+ account.reservations.create!(
68
+ byte_size: byte_size,
69
+ blob_id: blob_id&.to_s,
70
+ status: Reservation::STATUS_ACTIVE,
71
+ expires_at: now + ActiveStorageQuota.config.reservation_ttl
72
+ )
73
+ end
74
+ end
75
+
76
+ # Finds the account for this owner and scope, creating it if it is not
77
+ # there yet.
78
+ #
79
+ # Two first-ever reservations can reach the INSERT together; the unique
80
+ # index settles it and the loser re-finds the winning row. The INSERT sits
81
+ # in a savepoint because on PostgreSQL a constraint violation poisons the
82
+ # enclosing transaction, and this one is nested inside reserve!'s.
83
+ #
84
+ # Only RecordNotUnique is rescued: that is the one race expected here, and
85
+ # anything else is a real failure that must surface.
86
+ def find_or_create_for!(owner, scope_name = ActiveStorageQuota::DEFAULT_SCOPE)
87
+ scope_name = scope_name.to_s
88
+
89
+ find_by(owner: owner, scope_name: scope_name) ||
90
+ create_for!(owner, scope_name)
91
+ end
92
+
93
+ private
94
+ def create_for!(owner, scope_name)
95
+ transaction(requires_new: true) do
96
+ create!(owner: owner, scope_name: scope_name)
97
+ end
98
+ rescue ActiveRecord::RecordNotUnique
99
+ find_by!(owner: owner, scope_name: scope_name)
100
+ end
101
+
102
+ def validate_byte_size!(value)
103
+ unless value.is_a?(Integer) && !value.negative?
104
+ raise ArgumentError,
105
+ "byte size must be a non-negative Integer number of bytes, " \
106
+ "got #{value.inspect} (#{value.class})"
107
+ end
108
+
109
+ value
110
+ end
111
+ end
112
+
113
+ # Adds +byte_size+ to reserved_bytes, but only if it fits under
114
+ # +limit_bytes+. One statement: the capacity test is the WHERE clause of the
115
+ # very UPDATE that claims the bytes, so there is no window between checking
116
+ # and claiming for another transaction to slip through.
117
+ #
118
+ # A nil limit means unlimited, and the same claim runs without the test.
119
+ #
120
+ # Raises QuotaExceeded when nothing matched. Note this does not refresh the
121
+ # receiver: like any update_all, the in-memory object is now stale by
122
+ # design. Reload if you need its counters.
123
+ def claim_capacity!(byte_size, limit_bytes, now = Time.current)
124
+ rows = self.class.where(id: id)
125
+ rows = rows.where("used_bytes + reserved_bytes + ? <= ?", byte_size, limit_bytes) if limit_bytes
126
+
127
+ claimed = rows.update_all(
128
+ [ "reserved_bytes = reserved_bytes + ?, updated_at = ?", byte_size, now ]
129
+ ) == 1
130
+
131
+ return true if claimed
132
+
133
+ raise_claim_failure!(byte_size, limit_bytes)
134
+ end
135
+
136
+ # Adds +byte_size+ to used_bytes, but only if it fits under +limit_bytes+.
137
+ #
138
+ # Called only by Charge, in the transaction that inserted the charge row.
139
+ # Charge is the sole writer of used_bytes; nothing else in the gem adds to
140
+ # it, which is what keeps used_bytes == SUM(charges.byte_size) true by
141
+ # construction rather than by hope.
142
+ def claim_used_capacity!(byte_size, limit_bytes, now = Time.current)
143
+ rows = self.class.where(id: id)
144
+ rows = rows.where("used_bytes + reserved_bytes + ? <= ?", byte_size, limit_bytes) if limit_bytes
145
+
146
+ claimed = rows.update_all(
147
+ [ "used_bytes = used_bytes + ?, updated_at = ?", byte_size, now ]
148
+ ) == 1
149
+
150
+ return true if claimed
151
+
152
+ raise_claim_failure!(byte_size, limit_bytes)
153
+ end
154
+
155
+ # Subtracts a discarded charge's bytes from used_bytes.
156
+ #
157
+ # Guarded so a drifted counter cannot go negative, and so the drift is
158
+ # reported rather than silently absorbed. Called only by Charge, in the
159
+ # transaction that deleted the charge row.
160
+ def self.release_used_bytes!(quota_account_id, byte_size, now = Time.current)
161
+ affected = where(id: quota_account_id).where("used_bytes >= ?", byte_size)
162
+ .update_all([ "used_bytes = used_bytes - ?, updated_at = ?", byte_size, now ])
163
+
164
+ return true if affected == 1
165
+
166
+ raise CounterInvariantViolation,
167
+ "account #{quota_account_id} is charged fewer than #{byte_size} used " \
168
+ "bytes; refusing to lower the counter. Its counters have drifted " \
169
+ "from its charges."
170
+ end
171
+
172
+ private
173
+ def raise_claim_failure!(byte_size, limit_bytes)
174
+ # Re-read only here, on the failure path, so the exception can carry real
175
+ # numbers without costing the successful path an extra SELECT.
176
+ fresh = self.class.find_by(id: id)
177
+
178
+ if fresh.nil?
179
+ raise ActiveRecord::RecordNotFound,
180
+ "quota account #{id} no longer exists"
181
+ end
182
+
183
+ # An unlimited claim has no capacity test to fail, so a missed row can
184
+ # only mean the account went away between the UPDATE and this read.
185
+ if limit_bytes.nil?
186
+ raise ActiveRecord::RecordNotFound,
187
+ "quota account #{id} could not be updated"
188
+ end
189
+
190
+ raise QuotaExceeded.new(
191
+ owner: owner,
192
+ scope_name: scope_name,
193
+ requested_bytes: byte_size,
194
+ limit_bytes: limit_bytes,
195
+ used_bytes: fresh.used_bytes,
196
+ reserved_bytes: fresh.reserved_bytes
197
+ )
198
+ end
199
+ end
200
+ end
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_support/concern"
4
+
5
+ module ActiveStorageQuota
6
+ # Declares which quota owner pays for a model's attachments.
7
+ #
8
+ # class Contract < ApplicationRecord
9
+ # belongs_to :company
10
+ # storage_quota_owner :company
11
+ #
12
+ # has_many_attached :documents
13
+ # end
14
+ #
15
+ # Models that do not declare this are never charged. Models that do are always
16
+ # charged: an owner resolving to nil raises MissingQuotaOwner rather than
17
+ # quietly storing bytes nobody pays for.
18
+ module Attachable
19
+ extend ActiveSupport::Concern
20
+
21
+ # Extended onto ActiveRecord::Base so any model can declare its payer.
22
+ module Macro
23
+ # See ActiveStorageQuota::OwnerSource for the accepted forms.
24
+ #
25
+ # Subclasses may redeclare it without affecting the parent.
26
+ def storage_quota_owner(source)
27
+ include ActiveStorageQuota::Attachable
28
+
29
+ self.storage_quota_owner_source = ActiveStorageQuota::OwnerSource.new(source)
30
+ end
31
+ end
32
+
33
+ included do
34
+ class_attribute :storage_quota_owner_source, instance_accessor: false
35
+ end
36
+
37
+ # The quota owner charged for this record's attachments.
38
+ #
39
+ # Raises MissingQuotaOwner if it resolves to nil.
40
+ def storage_quota_owner
41
+ self.class.storage_quota_owner_source.resolve(self)
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,144 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_support/concern"
4
+
5
+ module ActiveStorageQuota
6
+ # Mixed into ActiveStorage::Attachment through the :active_storage_attachment
7
+ # load hook. No Active Storage internals are patched.
8
+ #
9
+ # Creation and destruction hook in at deliberately different points:
10
+ #
11
+ # after_create in the attachment's transaction -> fails CLOSED
12
+ # after_update in the attachment's transaction -> fails CLOSED
13
+ # after_destroy_commit after it has committed -> fails OPEN
14
+ #
15
+ # after_update covers attachment.update!(blob: other), which re-points an
16
+ # attachment at a different blob in place. That is an admission like a
17
+ # create, so a refusal must roll the update back with it.
18
+ #
19
+ # Failing closed on create is the whole point of a quota. Failing closed on
20
+ # delete would be far worse than useless: an account over its limit could not
21
+ # delete files to get back under it. So deletion accounting can never undo a
22
+ # deletion, and reports its failures instead.
23
+ #
24
+ # Not every removal is observable. Active Storage's purge and detach paths use
25
+ # delete / delete_all, which fire no callbacks at all, so attachment.purge is
26
+ # seen through the blob's destruction instead (see BlobAccounting) and detach
27
+ # is not seen at all. Those are reconciliation's job, not a reason to patch
28
+ # Rails.
29
+ module AttachmentAccounting
30
+ extend ActiveSupport::Concern
31
+
32
+ included do
33
+ after_create :record_storage_quota_charge
34
+ after_update :move_storage_quota_charge, if: :storage_quota_blob_swapped?
35
+ before_destroy :snapshot_storage_quota_accounting
36
+ after_destroy_commit :discard_storage_quota_charge
37
+ end
38
+
39
+ private
40
+ # Records whose model never declared storage_quota_owner are ignored
41
+ # entirely. That includes Active Storage's own internal attachments, such
42
+ # as a blob's preview_image.
43
+ def storage_quota_accountable?
44
+ record.class.respond_to?(:storage_quota_owner_source) &&
45
+ record.class.storage_quota_owner_source.present?
46
+ end
47
+
48
+ def record_storage_quota_charge
49
+ return unless storage_quota_accountable?
50
+
51
+ owner = record.storage_quota_owner
52
+ account = ActiveStorageQuota::Account.find_or_create_for!(owner)
53
+
54
+ ActiveStorageQuota::Charge.record!(
55
+ account: account,
56
+ blob_id: blob_id,
57
+ byte_size: blob.byte_size,
58
+ limit_bytes: owner.storage_limit
59
+ )
60
+ end
61
+
62
+ # Only a change of blob on the same record. Moving an attachment to a
63
+ # different record is not accounted here: the old charge belongs to the
64
+ # previous record's owner, and guessing it risks debiting the wrong
65
+ # account. Reconciliation repairs it.
66
+ def storage_quota_blob_swapped?
67
+ saved_change_to_blob_id? && !saved_change_to_record_id? && !saved_change_to_record_type?
68
+ end
69
+
70
+ def move_storage_quota_charge
71
+ return unless storage_quota_accountable?
72
+
73
+ owner = record.storage_quota_owner
74
+ account = ActiveStorageQuota::Account.find_or_create_for!(owner)
75
+
76
+ ActiveStorageQuota::Charge.move!(
77
+ account: account,
78
+ from_blob_id: blob_id_before_last_save,
79
+ to_blob_id: blob_id,
80
+ byte_size: blob.byte_size,
81
+ limit_bytes: owner.storage_limit
82
+ )
83
+ end
84
+
85
+ # Captures the accounting identity while the owning record still exists.
86
+ #
87
+ # By the time after_destroy_commit runs, `contract.destroy!` will have
88
+ # taken the Contract with it, so resolving the owner there would fail. The
89
+ # snapshot is plain scalars, never Active Record objects.
90
+ def snapshot_storage_quota_accounting
91
+ @storage_quota_snapshot = nil
92
+ @storage_quota_snapshot_error = nil
93
+
94
+ return unless storage_quota_accountable?
95
+
96
+ owner = record.class.storage_quota_owner_source.resolve_safely(record)
97
+ return if owner.nil?
98
+
99
+ account = owner.storage_quota_account
100
+ return if account.nil?
101
+
102
+ @storage_quota_snapshot = {
103
+ quota_account_id: account.id,
104
+ blob_id: blob_id.to_s,
105
+ byte_size: blob&.byte_size
106
+ }
107
+ rescue StandardError => e
108
+ # Malformed historical data must not make a file undeletable.
109
+ @storage_quota_snapshot_error = e
110
+ nil
111
+ end
112
+
113
+ def discard_storage_quota_charge
114
+ if @storage_quota_snapshot_error
115
+ report_storage_quota_failure(@storage_quota_snapshot_error)
116
+ return
117
+ end
118
+
119
+ return if @storage_quota_snapshot.nil?
120
+
121
+ ActiveStorageQuota::Charge.discard!(
122
+ quota_account_id: @storage_quota_snapshot[:quota_account_id],
123
+ blob_id: @storage_quota_snapshot[:blob_id]
124
+ )
125
+ rescue StandardError => e
126
+ # The attachment is already gone. Re-raising here would tell the caller
127
+ # the deletion failed when it did not.
128
+ report_storage_quota_failure(e)
129
+ end
130
+
131
+ def report_storage_quota_failure(exception)
132
+ snapshot = @storage_quota_snapshot || {}
133
+
134
+ ActiveStorageQuota.accounting_failed(
135
+ operation: "attachment_destroy",
136
+ exception: exception,
137
+ quota_account_id: snapshot[:quota_account_id],
138
+ blob_id: snapshot[:blob_id] || blob_id.to_s,
139
+ record_type: record_type,
140
+ record_id: record_id
141
+ )
142
+ end
143
+ end
144
+ end
@@ -0,0 +1,62 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveStorageQuota
4
+ module Audit
5
+ # Account#used_bytes against the charges it aggregates.
6
+ #
7
+ # The narrower of the two audits, and the one that can always run: both
8
+ # tables belong to this gem and share a connection by construction, so this
9
+ # is one adapter-neutral aggregate with no Active Storage involvement at
10
+ # all. It answers "is the denormalized counter consistent with the ledger",
11
+ # not "is the ledger right" -- that is the ledger audit's job.
12
+ class CounterAudit
13
+ BATCH_SIZE = 1_000
14
+
15
+ def initialize(result:)
16
+ @result = result
17
+ end
18
+
19
+ def run
20
+ each_account_batch do |accounts|
21
+ expected = expected_used_bytes(accounts.map(&:first))
22
+
23
+ accounts.each do |id, stored|
24
+ actual = expected.fetch(id, 0)
25
+ next if stored == actual
26
+
27
+ @result.record(
28
+ :account_counter_mismatch,
29
+ quota_account_id: id,
30
+ stored_used_bytes: stored,
31
+ expected_used_bytes: actual,
32
+ difference: stored - actual
33
+ )
34
+ end
35
+ end
36
+ end
37
+
38
+ private
39
+ # Keyset paging on the primary key. No offsets, so the cost per batch
40
+ # does not grow with how far in we are.
41
+ def each_account_batch
42
+ last_id = nil
43
+
44
+ loop do
45
+ scope = Account.order(:id).limit(BATCH_SIZE)
46
+ scope = scope.where(Account.arel_table[:id].gt(last_id)) if last_id
47
+ batch = scope.pluck(:id, :used_bytes)
48
+ break if batch.empty?
49
+
50
+ yield batch
51
+ last_id = batch.last.first
52
+ end
53
+ end
54
+
55
+ def expected_used_bytes(account_ids)
56
+ Charge.where(quota_account_id: account_ids)
57
+ .group(:quota_account_id)
58
+ .sum(:byte_size)
59
+ end
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveStorageQuota
4
+ module Audit
5
+ # One thing the audit noticed. A value object of scalars: findings are
6
+ # reported, logged and serialised, never acted on.
7
+ Finding = Struct.new(:category, :details, keyword_init: true) do
8
+ def to_h
9
+ { category: category }.merge(details)
10
+ end
11
+
12
+ def to_s
13
+ "#{category}: #{details.map { |k, v| "#{k}=#{v.inspect}" }.join(' ')}"
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,164 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveStorageQuota
4
+ module Audit
5
+ # Charges against the Active Storage attachments and blobs they claim to
6
+ # describe.
7
+ #
8
+ # Two passes, because neither alone sees everything:
9
+ #
10
+ # attachment-driven finds charges that should exist but do not, and
11
+ # reference counts that have drifted
12
+ # charge-driven finds charges whose attachment or blob is gone
13
+ #
14
+ # Both enumerate by blob id rather than by primary key order or a
15
+ # high-water mark. Grouping by blob keeps every (account, blob) group inside
16
+ # one batch, so counts are exact, and it makes no assumption that ids are
17
+ # sequential -- Active Storage primary keys may be UUIDs.
18
+ class LedgerAudit
19
+ BATCH_SIZE = 500
20
+
21
+ def initialize(result:, same_connection:)
22
+ @result = result
23
+ @same_connection = same_connection
24
+ @resolver = ActiveStorageQuota::OwnerResolver.new(result: result)
25
+ @quiet_resolver = ActiveStorageQuota::OwnerResolver.new(result: result, record_findings: false)
26
+ end
27
+
28
+ def run
29
+ audit_attachments
30
+ audit_charges
31
+ end
32
+
33
+ private
34
+ attr_reader :result
35
+
36
+ # Pass one: walk the attachments, ask what the ledger should say.
37
+ def audit_attachments
38
+ each_blob_id_batch do |blob_ids|
39
+ pairs = @resolver.pairs_for_blobs(blob_ids)
40
+ next if pairs.empty?
41
+
42
+ charges = charges_for(blob_ids)
43
+ sizes = blob_sizes(blob_ids)
44
+
45
+ pairs.each do |(account_id, blob_id), attachment_count|
46
+ charge = charges[[ account_id, blob_id ]]
47
+
48
+ if charge.nil?
49
+ result.record(
50
+ :missing_charge,
51
+ quota_account_id: account_id, blob_id: blob_id,
52
+ expected_attachment_count: attachment_count,
53
+ byte_size: sizes[blob_id]
54
+ )
55
+ elsif charge[:attachment_count] != attachment_count
56
+ result.record(
57
+ :attachment_count_mismatch,
58
+ quota_account_id: account_id, blob_id: blob_id,
59
+ charge_id: charge[:id],
60
+ stored_attachment_count: charge[:attachment_count],
61
+ expected_attachment_count: attachment_count
62
+ )
63
+ end
64
+ end
65
+ end
66
+ end
67
+
68
+ # Pass two: walk the ledger, ask whether anything still backs it.
69
+ def audit_charges
70
+ each_charge_batch do |charges|
71
+ blob_ids = charges.map { |charge| charge[:blob_id] }.uniq
72
+ sizes = blob_sizes(blob_ids)
73
+ pairs = @quiet_resolver.pairs_for_blobs(blob_ids)
74
+
75
+ charges.each do |charge|
76
+ audit_charge_blob(charge, sizes)
77
+
78
+ next if pairs.key?([ charge[:quota_account_id], charge[:blob_id] ])
79
+
80
+ result.record(
81
+ :unexpected_charge,
82
+ quota_account_id: charge[:quota_account_id], charge_id: charge[:id],
83
+ blob_id: charge[:blob_id], byte_size: charge[:byte_size],
84
+ attachment_count: charge[:attachment_count]
85
+ )
86
+ end
87
+ end
88
+ end
89
+
90
+ def audit_charge_blob(charge, sizes)
91
+ unless sizes.key?(charge[:blob_id])
92
+ result.record(
93
+ :charge_missing_blob,
94
+ quota_account_id: charge[:quota_account_id], charge_id: charge[:id],
95
+ blob_id: charge[:blob_id], byte_size: charge[:byte_size]
96
+ )
97
+ return
98
+ end
99
+
100
+ actual = sizes[charge[:blob_id]]
101
+ return if actual == charge[:byte_size]
102
+
103
+ # Which figure is right is a repair decision, not an audit one.
104
+ result.record(
105
+ :charge_byte_size_mismatch,
106
+ quota_account_id: charge[:quota_account_id], charge_id: charge[:id],
107
+ blob_id: charge[:blob_id],
108
+ charge_byte_size: charge[:byte_size], blob_byte_size: actual
109
+ )
110
+ end
111
+
112
+ # Keyset paging over distinct blob ids. String comparison works for
113
+ # bigint, uuid and ULID keys alike, and blob_id is indexed on
114
+ # attachments (Active Storage's t.references :blob creates it).
115
+ def each_blob_id_batch
116
+ last = nil
117
+
118
+ loop do
119
+ scope = ActiveStorage::Attachment.order(:blob_id).limit(BATCH_SIZE)
120
+ scope = scope.where(ActiveStorage::Attachment.arel_table[:blob_id].gt(last)) if last
121
+ blob_ids = scope.distinct.pluck(:blob_id)
122
+ break if blob_ids.empty?
123
+
124
+ yield blob_ids
125
+ last = blob_ids.last
126
+ end
127
+ end
128
+
129
+ def each_charge_batch
130
+ last_id = nil
131
+
132
+ loop do
133
+ scope = Charge.order(:id).limit(BATCH_SIZE)
134
+ scope = scope.where(Charge.arel_table[:id].gt(last_id)) if last_id
135
+ rows = scope.pluck(:id, :quota_account_id, :blob_id, :byte_size, :attachment_count)
136
+ break if rows.empty?
137
+
138
+ yield rows.map { |id, account_id, blob_id, byte_size, count|
139
+ { id: id, quota_account_id: account_id, blob_id: blob_id.to_s,
140
+ byte_size: byte_size, attachment_count: count }
141
+ }
142
+ last_id = rows.last.first
143
+ end
144
+ end
145
+
146
+ def charges_for(blob_ids)
147
+ Charge.where(blob_id: blob_ids.map(&:to_s))
148
+ .pluck(:id, :quota_account_id, :blob_id, :attachment_count)
149
+ .each_with_object({}) do |(id, account_id, blob_id, count), map|
150
+ map[[ account_id, blob_id.to_s ]] = { id: id, attachment_count: count }
151
+ end
152
+ end
153
+
154
+ # Reading blobs is the one step that may cross connections. It is an
155
+ # ordinary batched query either way; only the consistency label differs,
156
+ # because values read on another connection can shift between reads.
157
+ def blob_sizes(blob_ids)
158
+ ActiveStorage::Blob.where(id: blob_ids)
159
+ .pluck(:id, :byte_size)
160
+ .each_with_object({}) { |(id, size), map| map[id.to_s] = size }
161
+ end
162
+ end
163
+ end
164
+ end