active_storage_quota 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +52 -0
- data/LICENSE.txt +21 -0
- data/README.md +404 -0
- data/app/controllers/active_storage_quota/direct_uploads_controller.rb +96 -0
- data/db/migrate/20260917000001_create_active_storage_quota_tables.rb +72 -0
- data/db/migrate/20260918000001_create_active_storage_quota_charges.rb +45 -0
- data/lib/active_storage_quota/account.rb +200 -0
- data/lib/active_storage_quota/attachable.rb +44 -0
- data/lib/active_storage_quota/attachment_accounting.rb +144 -0
- data/lib/active_storage_quota/audit/counter_audit.rb +62 -0
- data/lib/active_storage_quota/audit/finding.rb +17 -0
- data/lib/active_storage_quota/audit/ledger_audit.rb +164 -0
- data/lib/active_storage_quota/audit/result.rb +106 -0
- data/lib/active_storage_quota/audit.rb +75 -0
- data/lib/active_storage_quota/backfill/cursor.rb +64 -0
- data/lib/active_storage_quota/backfill/result.rb +146 -0
- data/lib/active_storage_quota/backfill.rb +310 -0
- data/lib/active_storage_quota/blob_accounting.rb +44 -0
- data/lib/active_storage_quota/charge.rb +379 -0
- data/lib/active_storage_quota/configuration.rb +37 -0
- data/lib/active_storage_quota/definition.rb +100 -0
- data/lib/active_storage_quota/engine.rb +28 -0
- data/lib/active_storage_quota/errors.rb +115 -0
- data/lib/active_storage_quota/owner.rb +154 -0
- data/lib/active_storage_quota/owner_resolver.rb +402 -0
- data/lib/active_storage_quota/owner_source.rb +88 -0
- data/lib/active_storage_quota/reconciliation/budget.rb +70 -0
- data/lib/active_storage_quota/reconciliation/counter_reconciler.rb +78 -0
- data/lib/active_storage_quota/reconciliation/ledger_reconciler.rb +250 -0
- data/lib/active_storage_quota/reconciliation/result.rb +107 -0
- data/lib/active_storage_quota/reconciliation.rb +72 -0
- data/lib/active_storage_quota/record.rb +20 -0
- data/lib/active_storage_quota/reservation.rb +297 -0
- data/lib/active_storage_quota/version.rb +5 -0
- data/lib/active_storage_quota.rb +207 -0
- data/lib/tasks/active_storage_quota.rake +11 -0
- data/lib/tasks/active_storage_quota_audit.rake +15 -0
- data/lib/tasks/active_storage_quota_backfill.rake +104 -0
- data/lib/tasks/active_storage_quota_reconcile.rake +100 -0
- metadata +150 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 97ca05b6982dba442274c20efc15665e85f174879a9f695b31be021b277bd89e
|
|
4
|
+
data.tar.gz: '09e144b62a35850cdf84e6994e3f9dcc0fb96805dc5dd02686ab30c0195f9eaf'
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 05d7b59f2dabd66b623d25c0b0391484a0b41e8279ec7ed9685e93cbe32bd75b8e477db9ed34c43391ef2959c5b9d1fd3243e26b16d80ea60007336677186ed3
|
|
7
|
+
data.tar.gz: 2b29d72c64a566dc27296d93949071bfd69421033c9f85c76cd4f359e9948645b9794e117fcecd4a435652f52419a82a048930d2f40de76a84b900266f748851
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. This project follows
|
|
4
|
+
[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
5
|
+
|
|
6
|
+
## [0.1.0]
|
|
7
|
+
|
|
8
|
+
First release.
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- `has_storage_quota` declares a quota owner, with limits given as an Integer,
|
|
13
|
+
a Symbol, a callable, or `nil` for unlimited. Readers: `storage_limit`,
|
|
14
|
+
`storage_usage`, `storage_reserved`, `storage_remaining`,
|
|
15
|
+
`storage_quota_exceeded?`.
|
|
16
|
+
- `storage_quota_owner` declares which owner a model's attachments are charged
|
|
17
|
+
to. Works for `has_one_attached` and `has_many_attached` alike, and for every
|
|
18
|
+
attachment name on the model.
|
|
19
|
+
- Active Storage accounting through documented load hooks, with no
|
|
20
|
+
monkey-patching. The same blob is charged once per owner and reference
|
|
21
|
+
counted; different owners are charged independently. Records hidden by a
|
|
22
|
+
`default_scope` still count, and re-pointing an attachment at another blob
|
|
23
|
+
moves its charge.
|
|
24
|
+
- Race-safe admission. Capacity is tested and claimed in a single conditional
|
|
25
|
+
`UPDATE`, so concurrent uploads across processes cannot overshoot a quota.
|
|
26
|
+
Refused uploads raise `ActiveStorageQuota::QuotaExceeded` and persist nothing.
|
|
27
|
+
- Storage reservations for uploads that span requests, with a hard expiry,
|
|
28
|
+
`ActiveStorageQuota.release_expired_reservations!` and a rake task. Destroying
|
|
29
|
+
a blob releases any reservations naming it.
|
|
30
|
+
- `ActiveStorageQuota::DirectUploadsController`, a quota-aware direct-upload
|
|
31
|
+
endpoint. Subclass it and route to it; the response is Rails' own JSON, so
|
|
32
|
+
stock `@rails/activestorage` needs no changes. A full quota returns
|
|
33
|
+
`409 Conflict` with `{"error":"storage_quota_exceeded"}`.
|
|
34
|
+
- `ActiveStorageQuota.audit`, read-only inspection reporting eleven categories
|
|
35
|
+
of drift with exact totals and bounded samples. Performs no writes of any
|
|
36
|
+
kind.
|
|
37
|
+
- `reconcile_counters!`, `reconcile_ledger!` and `reconcile!`, repairing
|
|
38
|
+
counters and the charge ledger under short locked transactions. Dry runs plan
|
|
39
|
+
through the same logic they apply. Removing charges is opt-in and bounded by
|
|
40
|
+
a circuit breaker.
|
|
41
|
+
- `ActiveStorageQuota::Backfill`, building the initial ledger for an application
|
|
42
|
+
adopting the gem with existing attachments. Strictly additive, idempotent and
|
|
43
|
+
resumable through an opaque cursor.
|
|
44
|
+
- Rake tasks: `audit`, `reconcile`, `reconcile:apply`, `backfill`,
|
|
45
|
+
`backfill:apply`, `release_expired_reservations`, `install:migrations`.
|
|
46
|
+
|
|
47
|
+
### Supported
|
|
48
|
+
|
|
49
|
+
- Ruby >= 3.1; Active Record, Active Storage and Railties >= 7.1 and < 8.2.
|
|
50
|
+
- CI covers Rails 7.1, 7.2, 8.0 and 8.1.
|
|
51
|
+
- PostgreSQL is the primary production target and where concurrency is tested;
|
|
52
|
+
SQLite and MySQL are supported.
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Mubasher Khan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,404 @@
|
|
|
1
|
+
# ActiveStorageQuota
|
|
2
|
+
|
|
3
|
+
Race-safe storage quotas for Rails Active Storage.
|
|
4
|
+
|
|
5
|
+
## The problem
|
|
6
|
+
|
|
7
|
+
Rails Active Storage stores files but does not enforce how many bytes an owner
|
|
8
|
+
may store. The naive check is a race:
|
|
9
|
+
|
|
10
|
+
```ruby
|
|
11
|
+
if company.storage_remaining >= file.size # two requests both read 100MB free
|
|
12
|
+
company.documents.attach(file) # both attach 80MB; you are over quota
|
|
13
|
+
end
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
This gem replaces check-then-act with an atomic capacity claim, so concurrent
|
|
17
|
+
uploads across multiple processes and servers cannot overshoot a quota.
|
|
18
|
+
|
|
19
|
+
## Requirements
|
|
20
|
+
|
|
21
|
+
- Ruby >= 3.1
|
|
22
|
+
- Active Record, Active Storage and Railties >= 7.1 and < 8.2
|
|
23
|
+
- CI covers Rails 7.1, 7.2, 8.0 and 8.1
|
|
24
|
+
- PostgreSQL is the primary production target and is where the concurrency
|
|
25
|
+
guarantees are tested.
|
|
26
|
+
- SQLite is supported for non-concurrency behavior and is covered by the test
|
|
27
|
+
suite.
|
|
28
|
+
- Other Active Record adapters are not currently part of the CI matrix.
|
|
29
|
+
|
|
30
|
+
## Installation
|
|
31
|
+
|
|
32
|
+
```ruby
|
|
33
|
+
gem "active_storage_quota"
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
```sh
|
|
37
|
+
bundle install
|
|
38
|
+
bin/rails active_storage_quota:install:migrations
|
|
39
|
+
bin/rails db:migrate
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Usage
|
|
43
|
+
|
|
44
|
+
Declare who holds a quota, and which records' attachments count against it.
|
|
45
|
+
|
|
46
|
+
```ruby
|
|
47
|
+
class Company < ApplicationRecord
|
|
48
|
+
has_storage_quota limit: 5.gigabytes
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
class Property < ApplicationRecord
|
|
52
|
+
belongs_to :company
|
|
53
|
+
|
|
54
|
+
storage_quota_owner :company
|
|
55
|
+
|
|
56
|
+
has_many_attached :photos
|
|
57
|
+
has_one_attached :file
|
|
58
|
+
end
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
That is the whole setup. Attaching now works exactly as it always did, except
|
|
62
|
+
that an upload which would exceed the quota raises
|
|
63
|
+
`ActiveStorageQuota::QuotaExceeded` and is not persisted.
|
|
64
|
+
|
|
65
|
+
```ruby
|
|
66
|
+
property.file.attach(io: pdf, filename: "contract.pdf", content_type: "application/pdf")
|
|
67
|
+
property.photos.attach(io: png, filename: "front.png", content_type: "image/png")
|
|
68
|
+
|
|
69
|
+
company.storage_usage # => 1_234_567 bytes currently stored
|
|
70
|
+
company.storage_limit # => 5368709120 bytes, or nil when unlimited
|
|
71
|
+
company.storage_remaining # => bytes left, floored at 0, or nil when unlimited
|
|
72
|
+
company.storage_reserved # => bytes held by uploads in flight
|
|
73
|
+
company.storage_quota_exceeded? # => false
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Every byte value is an `Integer`.
|
|
77
|
+
|
|
78
|
+
### What counts
|
|
79
|
+
|
|
80
|
+
- **Both attachment macros are supported.** `has_many_attached` and
|
|
81
|
+
`has_one_attached` behave identically, and every attachment name on a model
|
|
82
|
+
uses that model's declared `storage_quota_owner`.
|
|
83
|
+
- **The same blob attached twice to one owner is charged once.** Active Storage
|
|
84
|
+
stores one object, so the owner pays for it once; the charge is reference
|
|
85
|
+
counted and released when the last attachment goes.
|
|
86
|
+
- **The same blob attached by two different owners is charged to each in
|
|
87
|
+
full.** A quota is an entitlement, not a bill for physical bytes, so one
|
|
88
|
+
tenant's usage never depends on what another tenant happens to store.
|
|
89
|
+
- **Soft-deleted records still count.** A record hidden by its model's
|
|
90
|
+
`default_scope` still owns its attachments, so they stay charged until the
|
|
91
|
+
attachments themselves are removed.
|
|
92
|
+
- **Swapping a blob in place moves the charge.** `attachment.update!(blob: other)`
|
|
93
|
+
releases the old blob and admits the new one in the same transaction; if the
|
|
94
|
+
new blob does not fit, the update is rolled back.
|
|
95
|
+
|
|
96
|
+
### Dynamic and unlimited limits
|
|
97
|
+
|
|
98
|
+
`limit:` takes an Integer, a Symbol naming a method on the owner, a callable, or
|
|
99
|
+
`nil`.
|
|
100
|
+
|
|
101
|
+
```ruby
|
|
102
|
+
has_storage_quota limit: :plan_storage_limit
|
|
103
|
+
has_storage_quota limit: ->(company) { company.subscription.storage_limit }
|
|
104
|
+
has_storage_quota limit: nil # unlimited: still accounted, never refused
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Nothing is coerced. A limit of `"5000"` or `5.0` raises
|
|
108
|
+
`ActiveStorageQuota::ConfigurationError` rather than being rounded off.
|
|
109
|
+
|
|
110
|
+
### Reads are advisory
|
|
111
|
+
|
|
112
|
+
`storage_usage`, `storage_remaining` and the other read helpers describe the
|
|
113
|
+
quota at the instant they run. They suit UI such as a usage bar, but must not be
|
|
114
|
+
used as a check-then-act admission guard.
|
|
115
|
+
|
|
116
|
+
Admission is enforced by the gem's write paths — attaching a file, and issuing
|
|
117
|
+
a direct-upload URL — where the capacity test and the claim are one atomic
|
|
118
|
+
statement.
|
|
119
|
+
|
|
120
|
+
## Direct uploads
|
|
121
|
+
|
|
122
|
+
Browser-to-storage direct uploads bypass your server, so the quota has to be
|
|
123
|
+
checked when the browser asks for an upload URL. Subclass the gem's controller,
|
|
124
|
+
say who pays, route to it, and point your file inputs at that route. The gem
|
|
125
|
+
draws no routes of its own.
|
|
126
|
+
|
|
127
|
+
```ruby
|
|
128
|
+
# app/controllers/quota_direct_uploads_controller.rb
|
|
129
|
+
class QuotaDirectUploadsController < ActiveStorageQuota::DirectUploadsController
|
|
130
|
+
include Authentication
|
|
131
|
+
before_action :authenticate!
|
|
132
|
+
|
|
133
|
+
private
|
|
134
|
+
def storage_quota_owner
|
|
135
|
+
Current.company
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
```ruby
|
|
141
|
+
# config/routes.rb
|
|
142
|
+
post "/direct_uploads",
|
|
143
|
+
to: "quota_direct_uploads#create",
|
|
144
|
+
as: :quota_direct_uploads
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
```erb
|
|
148
|
+
<%= form.file_field :documents,
|
|
149
|
+
multiple: true,
|
|
150
|
+
data: { direct_upload_url: quota_direct_uploads_url } %>
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
The `data-direct-upload-url` attribute is all `@rails/activestorage` looks for.
|
|
154
|
+
Do not also pass `direct_upload: true`: Rails would then add a second
|
|
155
|
+
`data-direct-upload-url` pointing at its own endpoint, which issues upload URLs
|
|
156
|
+
without reserving quota.
|
|
157
|
+
|
|
158
|
+
That is the whole setup. The response is Rails' own direct-upload JSON, so
|
|
159
|
+
stock `@rails/activestorage` needs no custom JavaScript beyond using this
|
|
160
|
+
endpoint — there is no reservation token and nothing of ours in blob metadata.
|
|
161
|
+
Capacity is held against the owner's quota when the URL is issued, and becomes
|
|
162
|
+
usage when the signed blob id is attached.
|
|
163
|
+
|
|
164
|
+
### Authentication is yours
|
|
165
|
+
|
|
166
|
+
`ActiveStorageQuota::DirectUploadsController` inherits Active Storage's
|
|
167
|
+
controller, which descends from `ActionController::Base` — **not** from your
|
|
168
|
+
`ApplicationController`. Its `before_action` filters do not run here. Sessions,
|
|
169
|
+
cookies and CSRF protection all work normally.
|
|
170
|
+
|
|
171
|
+
`storage_quota_owner` is therefore the authorization boundary, and your
|
|
172
|
+
application must establish the authenticated tenant before it is trusted.
|
|
173
|
+
Include your own authentication concern, as above.
|
|
174
|
+
|
|
175
|
+
Returning `nil` from `storage_quota_owner`, or an object that does not declare
|
|
176
|
+
`has_storage_quota`, raises. Those are wiring mistakes, not request outcomes,
|
|
177
|
+
and the gem deliberately does not dress them up as a friendly 4xx.
|
|
178
|
+
|
|
179
|
+
### When the quota is full
|
|
180
|
+
|
|
181
|
+
```http
|
|
182
|
+
409 Conflict
|
|
183
|
+
{"error":"storage_quota_exceeded"}
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
409 rather than 413: the request is well formed and small — the file is not in
|
|
187
|
+
it — but it conflicts with the account's current state, and the user can resolve
|
|
188
|
+
it by freeing space and retrying. Override `quota_exceeded_status` in your
|
|
189
|
+
subclass if you disagree. No quota figures are returned; stock
|
|
190
|
+
`@rails/activestorage` discards error bodies and surfaces only the status.
|
|
191
|
+
|
|
192
|
+
### Abandoned and retried uploads
|
|
193
|
+
|
|
194
|
+
If the browser never uploads, or uploads but never attaches, the reservation is
|
|
195
|
+
released by `ActiveStorageQuota.release_expired_reservations!` and the blob by
|
|
196
|
+
Rails' own `active_storage:purge_unattached`. Two independent janitors; neither
|
|
197
|
+
needs the other.
|
|
198
|
+
|
|
199
|
+
A retried POST creates a second blob and a second hold, exactly as stock Active
|
|
200
|
+
Storage creates a second blob. Both expire. `reservation_ttl` controls how long
|
|
201
|
+
a duplicate or abandoned request holds capacity.
|
|
202
|
+
|
|
203
|
+
Destroying an unattached blob releases any holds naming it immediately, since a
|
|
204
|
+
destroyed blob can never be attached.
|
|
205
|
+
|
|
206
|
+
### Cross-database limitation
|
|
207
|
+
|
|
208
|
+
A quota refusal always rolls the blob back, because the exception propagates out
|
|
209
|
+
of the blob's transaction — that holds whether or not the quota tables share a
|
|
210
|
+
connection with Active Storage.
|
|
211
|
+
|
|
212
|
+
What separate databases cannot guarantee is the reverse: if the reservation
|
|
213
|
+
commits and an Active Storage-side operation then fails, the hold outlives the
|
|
214
|
+
blob. It is recovered by the reservation TTL. The gem does not attempt
|
|
215
|
+
distributed transactions or compensating deletes.
|
|
216
|
+
|
|
217
|
+
### A note on `byte_size`
|
|
218
|
+
|
|
219
|
+
`byte_size` originates from the direct-upload client. Whether the declared
|
|
220
|
+
length is actually enforced depends on the Active Storage service adapter: S3
|
|
221
|
+
binds it into the signed request and Disk verifies it against a signed token,
|
|
222
|
+
both verified. Other adapters need their own analysis. This gem does not verify
|
|
223
|
+
the stored object's physical size.
|
|
224
|
+
|
|
225
|
+
## Maintenance
|
|
226
|
+
|
|
227
|
+
Accounting is a denormalized ledger, so the gem ships tools to inspect and
|
|
228
|
+
repair it. All of them are safe to run against production.
|
|
229
|
+
|
|
230
|
+
```sh
|
|
231
|
+
bin/rails active_storage_quota:audit # read-only; writes nothing, ever
|
|
232
|
+
ALL=1 bin/rails active_storage_quota:reconcile # always a dry run; prints a plan
|
|
233
|
+
bin/rails active_storage_quota:release_expired_reservations
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
`audit` reports drift without touching anything, and exits non-zero when it
|
|
237
|
+
finds any, so it works as a cron or CI check. Repairs are a separate, explicitly
|
|
238
|
+
named task:
|
|
239
|
+
|
|
240
|
+
```sh
|
|
241
|
+
OWNER_TYPE=Company OWNER_ID=1 bin/rails active_storage_quota:reconcile:apply
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
Both reconcile tasks take `OWNER_TYPE` and `OWNER_ID` for one owner, or `ALL=1`
|
|
245
|
+
for every owner, and refuse to run given neither.
|
|
246
|
+
|
|
247
|
+
Removing charges that nothing backs any more is destructive, so it only happens
|
|
248
|
+
when you supply a budget (`MAX_CHARGE_DELETIONS` and `MAX_REMOVED_FRACTION` on
|
|
249
|
+
the rake tasks), and a circuit breaker stops a run that would remove more than
|
|
250
|
+
you allowed:
|
|
251
|
+
|
|
252
|
+
```ruby
|
|
253
|
+
ActiveStorageQuota.reconcile!(owner: company,
|
|
254
|
+
max_charge_deletions: 10,
|
|
255
|
+
max_removed_fraction: 0.25)
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
Reservations for uploads that were started and abandoned are released by
|
|
259
|
+
`release_expired_reservations`; run it from cron, a recurring job, or whatever
|
|
260
|
+
scheduler you already have. The gem does not ship one, and needs no Redis or
|
|
261
|
+
Sidekiq.
|
|
262
|
+
|
|
263
|
+
## Adopting active_storage_quota in an existing application
|
|
264
|
+
|
|
265
|
+
An application that already stores files has historical attachments the gem
|
|
266
|
+
knows nothing about. Backfill builds the initial ledger for them.
|
|
267
|
+
|
|
268
|
+
The order matters. Backfill writes charges but deliberately does **not** write
|
|
269
|
+
`used_bytes`, so counters understate the ledger until counter reconciliation
|
|
270
|
+
runs. Enforcing quotas against incomplete counters would admit uploads that
|
|
271
|
+
should have been refused, so enforcement stays off until the end.
|
|
272
|
+
|
|
273
|
+
```
|
|
274
|
+
1. Install and run the migrations.
|
|
275
|
+
|
|
276
|
+
2. Declare the macros:
|
|
277
|
+
|
|
278
|
+
class Company < ApplicationRecord
|
|
279
|
+
has_storage_quota limit: ->(company) {
|
|
280
|
+
Adoption.enforcing?(company) ? company.plan_limit : nil
|
|
281
|
+
}
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
class Contract < ApplicationRecord
|
|
285
|
+
belongs_to :company
|
|
286
|
+
storage_quota_owner :company
|
|
287
|
+
has_many_attached :documents
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
3. Deploy with limits resolving to nil.
|
|
291
|
+
Accounting is now active for new uploads; enforcement is not.
|
|
292
|
+
|
|
293
|
+
4. See what backfill would do:
|
|
294
|
+
bin/rails active_storage_quota:backfill
|
|
295
|
+
|
|
296
|
+
5. Create the historical charges, repeating until complete:
|
|
297
|
+
ALL=1 bin/rails active_storage_quota:backfill:apply
|
|
298
|
+
CURSOR=<printed cursor> ALL=1 bin/rails active_storage_quota:backfill:apply
|
|
299
|
+
|
|
300
|
+
6. Establish the counters from the finished ledger:
|
|
301
|
+
ActiveStorageQuota.reconcile_counters!(owner: company)
|
|
302
|
+
|
|
303
|
+
7. Check the result:
|
|
304
|
+
bin/rails active_storage_quota:audit
|
|
305
|
+
|
|
306
|
+
8. Repair anything the audit flags as repairable:
|
|
307
|
+
ActiveStorageQuota.reconcile!(owner: company)
|
|
308
|
+
|
|
309
|
+
9. Resolve the report-only findings by hand: missing or invalid quota owners,
|
|
310
|
+
unresolvable record types, byte size mismatches.
|
|
311
|
+
|
|
312
|
+
10. Switch limits to their real values, one tenant at a time.
|
|
313
|
+
|
|
314
|
+
11. bin/rails active_storage_quota:audit # expect it clean
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
`backfill:apply` refuses to start while any owner still has a real limit,
|
|
318
|
+
precisely because step 6 has not happened yet. Pass
|
|
319
|
+
`ALLOW_ACTIVE_ENFORCEMENT=1` only if you understand that consequence.
|
|
320
|
+
|
|
321
|
+
### The one exception to the ledger invariant
|
|
322
|
+
|
|
323
|
+
Normal operation always holds:
|
|
324
|
+
|
|
325
|
+
```ruby
|
|
326
|
+
account.used_bytes == account.charges.sum(:byte_size)
|
|
327
|
+
```
|
|
328
|
+
|
|
329
|
+
Backfill is the single sanctioned exception: between step 5 and step 6 charges
|
|
330
|
+
exist that `used_bytes` does not yet reflect. This is safe only while limits
|
|
331
|
+
resolve to nil, and step 6 closes it.
|
|
332
|
+
|
|
333
|
+
## Troubleshooting
|
|
334
|
+
|
|
335
|
+
### Rails 8.1 and json 3.x
|
|
336
|
+
|
|
337
|
+
Some Rails 8.1 releases have a compatibility problem between ActiveSupport's
|
|
338
|
+
JSON decoding, which Active Storage uses for blob metadata, and `json` 3.x.
|
|
339
|
+
|
|
340
|
+
If Active Storage operations fail with an error like:
|
|
341
|
+
|
|
342
|
+
```
|
|
343
|
+
wrong number of arguments (given 2, expected 1)
|
|
344
|
+
```
|
|
345
|
+
|
|
346
|
+
pin the host application to:
|
|
347
|
+
|
|
348
|
+
```ruby
|
|
349
|
+
gem "json", "< 3"
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
This is an upstream Rails/json incompatibility, not something
|
|
353
|
+
`active_storage_quota` causes or can fix, and it does not affect every Rails 8.1
|
|
354
|
+
installation.
|
|
355
|
+
|
|
356
|
+
## Known limitations
|
|
357
|
+
|
|
358
|
+
- **`detach` is not observed.** Active Storage's `detach` and `purge` paths
|
|
359
|
+
delete attachment rows without firing callbacks, so a `detach` leaves a stale
|
|
360
|
+
charge. `bin/rails active_storage_quota:audit` reports it and reconciliation
|
|
361
|
+
repairs it. Purging is observed through the blob's destruction. Moving an
|
|
362
|
+
attachment to a different record (`attachment.update!(record: other)`) is not
|
|
363
|
+
observed either, and is repaired the same way.
|
|
364
|
+
- **Replacing a `has_one_attached` file can briefly count both files.** Active
|
|
365
|
+
Storage creates the replacement attachment before removing the previous one,
|
|
366
|
+
so if the two files together exceed the remaining quota the replacement is
|
|
367
|
+
refused, even when the new file alone would fit. The previous file stays
|
|
368
|
+
attached and charged.
|
|
369
|
+
- **Variants and previews are not counted.** Generated variant and preview
|
|
370
|
+
storage is not represented as an attachment owned by your quota-bearing
|
|
371
|
+
application record, so those generated bytes are not included in quota usage.
|
|
372
|
+
- **Cross-database accounting is not atomic.** When the quota tables live on a
|
|
373
|
+
different connection from Active Storage, Rails cannot make the two writes one
|
|
374
|
+
transaction; accounting becomes eventually consistent and audit/reconciliation
|
|
375
|
+
are the guarantee.
|
|
376
|
+
- **Direct-upload `byte_size` comes from the client.** Whether the declared
|
|
377
|
+
length is enforced depends on the storage adapter: S3 binds it into the signed
|
|
378
|
+
request and Disk verifies it against a signed token. Other adapters need their
|
|
379
|
+
own analysis.
|
|
380
|
+
|
|
381
|
+
## Development
|
|
382
|
+
|
|
383
|
+
```sh
|
|
384
|
+
bin/setup # or: bundle install
|
|
385
|
+
bundle exec rspec
|
|
386
|
+
bundle exec rubocop
|
|
387
|
+
```
|
|
388
|
+
|
|
389
|
+
Against a specific Rails version:
|
|
390
|
+
|
|
391
|
+
```sh
|
|
392
|
+
BUNDLE_GEMFILE=gemfiles/rails_7.1.gemfile bundle install
|
|
393
|
+
BUNDLE_GEMFILE=gemfiles/rails_7.1.gemfile bundle exec rspec
|
|
394
|
+
```
|
|
395
|
+
|
|
396
|
+
Concurrency specs require PostgreSQL and are skipped on other adapters:
|
|
397
|
+
|
|
398
|
+
```sh
|
|
399
|
+
DATABASE_URL=postgres://localhost/asq_test bundle exec rspec
|
|
400
|
+
```
|
|
401
|
+
|
|
402
|
+
## License
|
|
403
|
+
|
|
404
|
+
MIT. See [LICENSE.txt](LICENSE.txt).
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveStorageQuota
|
|
4
|
+
# A quota-aware replacement for Rails' direct-upload endpoint.
|
|
5
|
+
#
|
|
6
|
+
# Subclass it, say who pays, and point your direct uploads at it:
|
|
7
|
+
#
|
|
8
|
+
# class QuotaDirectUploadsController < ActiveStorageQuota::DirectUploadsController
|
|
9
|
+
# include Authentication
|
|
10
|
+
# before_action :authenticate!
|
|
11
|
+
#
|
|
12
|
+
# private
|
|
13
|
+
# def storage_quota_owner
|
|
14
|
+
# Current.company
|
|
15
|
+
# end
|
|
16
|
+
# end
|
|
17
|
+
#
|
|
18
|
+
# post "/direct_uploads", to: "quota_direct_uploads#create"
|
|
19
|
+
#
|
|
20
|
+
# The gem draws no routes and replaces nothing: opting in is subclassing and
|
|
21
|
+
# routing. The browser contract is unchanged, so stock @rails/activestorage
|
|
22
|
+
# needs no modification beyond the endpoint URL -- there is no reservation
|
|
23
|
+
# token, and nothing of ours goes into blob metadata.
|
|
24
|
+
#
|
|
25
|
+
# Only +create+ is overridden. +blob_args+ differs between Rails 7.x
|
|
26
|
+
# (require/permit) and 8.x (params.expect) and +direct_upload_json+ defines the
|
|
27
|
+
# response contract, so both stay Rails' responsibility.
|
|
28
|
+
#
|
|
29
|
+
# ## Authentication
|
|
30
|
+
#
|
|
31
|
+
# This inherits ActiveStorage::BaseController, which descends from
|
|
32
|
+
# ActionController::Base -- *not* from the application's ApplicationController.
|
|
33
|
+
# Its before_action filters do not run here. Sessions and cookies do work, and
|
|
34
|
+
# CSRF protection is inherited unchanged.
|
|
35
|
+
#
|
|
36
|
+
# +storage_quota_owner+ is therefore the authorization boundary, and the
|
|
37
|
+
# application must establish the authenticated tenant before it is trusted.
|
|
38
|
+
class DirectUploadsController < ActiveStorage::DirectUploadsController
|
|
39
|
+
def create
|
|
40
|
+
# Resolved first: a wiring problem should cost nothing, least of all a
|
|
41
|
+
# stray blob row.
|
|
42
|
+
owner = resolved_storage_quota_owner
|
|
43
|
+
|
|
44
|
+
payload = ActiveStorage::Blob.transaction do
|
|
45
|
+
blob = ActiveStorage::Blob.create_before_direct_upload!(**blob_args)
|
|
46
|
+
|
|
47
|
+
owner.reserve_storage!(blob.byte_size, blob: blob)
|
|
48
|
+
|
|
49
|
+
# Built inside the transaction on purpose. Signing the upload URL can
|
|
50
|
+
# fail, and doing it here means such a failure rolls back the blob and
|
|
51
|
+
# the reservation rather than leaving either behind.
|
|
52
|
+
direct_upload_json(blob)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
render json: payload
|
|
56
|
+
rescue ActiveStorageQuota::QuotaExceeded
|
|
57
|
+
render json: { error: "storage_quota_exceeded" }, status: quota_exceeded_status
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
private
|
|
61
|
+
# The quota owner charged for this upload. Host applications override it.
|
|
62
|
+
def storage_quota_owner
|
|
63
|
+
raise NotImplementedError,
|
|
64
|
+
"#{self.class.name} must define #storage_quota_owner, returning the " \
|
|
65
|
+
"record that declares has_storage_quota and pays for this upload"
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# 409 Conflict: the request is well formed, but it conflicts with the
|
|
69
|
+
# account's current state, and the user can resolve it by freeing space and
|
|
70
|
+
# resubmitting. 413 would claim the request body was too large, which it is
|
|
71
|
+
# not -- the file is not in this request.
|
|
72
|
+
def quota_exceeded_status
|
|
73
|
+
:conflict
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def resolved_storage_quota_owner
|
|
77
|
+
owner = storage_quota_owner
|
|
78
|
+
|
|
79
|
+
if owner.nil?
|
|
80
|
+
raise MissingQuotaOwner.new(
|
|
81
|
+
"#{self.class.name}#storage_quota_owner returned nil. Authenticate " \
|
|
82
|
+
"before this point: a missing owner is a wiring problem, not an " \
|
|
83
|
+
"expected request outcome."
|
|
84
|
+
)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
unless owner.respond_to?(:reserve_storage!)
|
|
88
|
+
raise ConfigurationError,
|
|
89
|
+
"#{self.class.name}#storage_quota_owner returned " \
|
|
90
|
+
"#{owner.class.name}, which does not declare has_storage_quota"
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
owner
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class CreateActiveStorageQuotaTables < ActiveRecord::Migration[7.1]
|
|
4
|
+
def change
|
|
5
|
+
create_table :active_storage_quota_accounts do |t|
|
|
6
|
+
# Polymorphic owner.
|
|
7
|
+
#
|
|
8
|
+
# owner_id is a string, not a bigint, so that owners keyed by bigint,
|
|
9
|
+
# UUID, ULID or any other primary key type can coexist in one
|
|
10
|
+
# application. Active Record casts transparently in both directions:
|
|
11
|
+
# an Integer id of 1 is written and queried as '1'.
|
|
12
|
+
#
|
|
13
|
+
# The cost is that ad-hoc SQL joining back to the owner needs a cast
|
|
14
|
+
# (`JOIN companies c ON c.id::text = a.owner_id`). The gem itself never
|
|
15
|
+
# joins here; it looks accounts up by owner and loads the owner through
|
|
16
|
+
# the polymorphic association.
|
|
17
|
+
#
|
|
18
|
+
# Column limits keep this unique index inside InnoDB's 3072-byte key
|
|
19
|
+
# limit for MySQL users: (255 + 255 + 100) * 4 bytes = 2440.
|
|
20
|
+
t.string :owner_type, null: false, limit: 255
|
|
21
|
+
t.string :owner_id, null: false, limit: 255
|
|
22
|
+
|
|
23
|
+
# Quota accounts are keyed by owner *and* scope so that named scopes
|
|
24
|
+
# ("legal", "crm", ...) can be added later without rebuilding this unique
|
|
25
|
+
# index on a live table. Nothing writes a value other than "default" yet.
|
|
26
|
+
t.string :scope_name, null: false, limit: 100, default: "default"
|
|
27
|
+
|
|
28
|
+
# Committed and in-flight byte counters. bigint because a 32-bit counter
|
|
29
|
+
# overflows at 2GB.
|
|
30
|
+
t.bigint :used_bytes, null: false, default: 0
|
|
31
|
+
t.bigint :reserved_bytes, null: false, default: 0
|
|
32
|
+
|
|
33
|
+
t.timestamps
|
|
34
|
+
|
|
35
|
+
t.index %i[owner_type owner_id scope_name],
|
|
36
|
+
unique: true, name: "index_asq_accounts_on_owner_and_scope"
|
|
37
|
+
|
|
38
|
+
# A counter going negative means a reservation was released twice. Fail
|
|
39
|
+
# loudly at the database rather than silently handing back free quota.
|
|
40
|
+
t.check_constraint "used_bytes >= 0",
|
|
41
|
+
name: "asq_accounts_used_bytes_non_negative"
|
|
42
|
+
t.check_constraint "reserved_bytes >= 0",
|
|
43
|
+
name: "asq_accounts_reserved_bytes_non_negative"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
create_table :active_storage_quota_reservations do |t|
|
|
47
|
+
t.references :quota_account, null: false, index: false,
|
|
48
|
+
foreign_key: { to_table: :active_storage_quota_accounts }
|
|
49
|
+
|
|
50
|
+
t.bigint :byte_size, null: false
|
|
51
|
+
|
|
52
|
+
# active -> committed | released. Terminal rows are kept: they make commit
|
|
53
|
+
# idempotent under job retries and double as the audit log for an account.
|
|
54
|
+
t.string :status, null: false, limit: 20, default: "active"
|
|
55
|
+
|
|
56
|
+
# A hard expiry. commit! guards on `expires_at > :now` and the sweeper on
|
|
57
|
+
# `expires_at <= :now` -- exact complements, so at any instant an active
|
|
58
|
+
# row is claimable by exactly one of them.
|
|
59
|
+
t.datetime :expires_at, null: false
|
|
60
|
+
|
|
61
|
+
t.timestamps
|
|
62
|
+
|
|
63
|
+
# Per-account lookups: reconciling reserved_bytes against active rows.
|
|
64
|
+
t.index %i[quota_account_id status]
|
|
65
|
+
# The sweeper's only query.
|
|
66
|
+
t.index %i[status expires_at]
|
|
67
|
+
|
|
68
|
+
t.check_constraint "byte_size >= 0",
|
|
69
|
+
name: "asq_reservations_byte_size_non_negative"
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|