activestorage-aws-record 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +55 -0
- data/LICENSE +21 -0
- data/PLAN.md +413 -0
- data/README.md +287 -0
- data/lib/active_storage/aws_record/attachable.rb +104 -0
- data/lib/active_storage/aws_record/attachment.rb +426 -0
- data/lib/active_storage/aws_record/blob.rb +417 -0
- data/lib/active_storage/aws_record/configuration.rb +62 -0
- data/lib/active_storage/aws_record/item.rb +86 -0
- data/lib/active_storage/aws_record/owner.rb +82 -0
- data/lib/active_storage/aws_record/persistence.rb +89 -0
- data/lib/active_storage/aws_record/railtie.rb +53 -0
- data/lib/active_storage/aws_record/relation.rb +163 -0
- data/lib/active_storage/aws_record/schema.rb +148 -0
- data/lib/active_storage/aws_record/tables.rb +82 -0
- data/lib/active_storage/aws_record/tasks.rake +15 -0
- data/lib/active_storage/aws_record/transaction.rb +132 -0
- data/lib/active_storage/aws_record/variant_record.rb +208 -0
- data/lib/active_storage/aws_record/version.rb +7 -0
- data/lib/active_storage/aws_record.rb +148 -0
- data/lib/activestorage-aws-record.rb +4 -0
- metadata +166 -0
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
require 'securerandom'
|
|
2
|
+
|
|
3
|
+
module ActiveStorage
|
|
4
|
+
module AwsRecord
|
|
5
|
+
# Generic aws-record plumbing shared by every model that participates in the
|
|
6
|
+
# Active Storage owner contract — both the gem's own entities and the
|
|
7
|
+
# application's owner models (which include {Owner}). It deliberately holds
|
|
8
|
+
# *no* single-table key logic (that lives in {Item}, mixed only into the
|
|
9
|
+
# gem's three entities), so an application owner keeps its own key schema.
|
|
10
|
+
#
|
|
11
|
+
# Provides: aws-record + GlobalID, the shared client, equality by class + id,
|
|
12
|
+
# +changed?+ → aws-record +dirty?+, raw attribute access, and helpers to mark
|
|
13
|
+
# an instance persisted/destroyed after a raw write.
|
|
14
|
+
module Persistence
|
|
15
|
+
extend ActiveSupport::Concern
|
|
16
|
+
|
|
17
|
+
included do
|
|
18
|
+
include Aws::Record
|
|
19
|
+
include GlobalID::Identification
|
|
20
|
+
|
|
21
|
+
# Defined here, after +include Aws::Record+, so it shadows aws-record's
|
|
22
|
+
# +find(opts)+ class method with the contract's +find(id)+. Single
|
|
23
|
+
# hash-key owners (the common case) use this as-is; the gem's own
|
|
24
|
+
# composite-key entities override it.
|
|
25
|
+
def self.find(id)
|
|
26
|
+
record = find_with_opts(key: { hash_key => id })
|
|
27
|
+
record || raise(ActiveStorage::RecordNotFound, "Couldn't find #{name} with id=#{id.inspect}")
|
|
28
|
+
rescue Aws::Record::Errors::KeyMissing
|
|
29
|
+
raise ActiveStorage::RecordNotFound, "Couldn't find #{name} with id=#{id.inspect}"
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
class_methods do
|
|
34
|
+
# @return [Aws::DynamoDB::Client] the shared client.
|
|
35
|
+
def dynamodb_client
|
|
36
|
+
ActiveStorage::AwsRecord.dynamodb_client
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def dynamodb_client
|
|
41
|
+
self.class.dynamodb_client
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def ==(other)
|
|
45
|
+
other.instance_of?(self.class) && id.present? && id == other.id
|
|
46
|
+
end
|
|
47
|
+
alias_method :eql?, :==
|
|
48
|
+
|
|
49
|
+
def hash
|
|
50
|
+
[self.class, id].hash
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Active Storage's generic owner path checks +changed?+; aws-record names
|
|
54
|
+
# the same concept +dirty?+.
|
|
55
|
+
def changed?
|
|
56
|
+
dirty?
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Raw attribute access via aws-record's data object (it has no AR-style
|
|
60
|
+
# +self[:attr]+), for use inside overridden accessors.
|
|
61
|
+
def read_attribute(name)
|
|
62
|
+
instance_variable_get('@data').get_attribute(name)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def write_attribute(name, value)
|
|
66
|
+
instance_variable_get('@data').set_attribute(name, value)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Mark this instance persisted after a raw (non-aws-record) write.
|
|
70
|
+
def mark_persisted!
|
|
71
|
+
data = instance_variable_get('@data')
|
|
72
|
+
data.new_record = false
|
|
73
|
+
data.destroyed = false
|
|
74
|
+
data.clean!
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Mark this instance destroyed after a raw delete.
|
|
78
|
+
def mark_destroyed!
|
|
79
|
+
instance_variable_get('@data').destroyed = true
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
private
|
|
83
|
+
|
|
84
|
+
def generate_uuid
|
|
85
|
+
SecureRandom.uuid
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
require 'rails/railtie'
|
|
2
|
+
|
|
3
|
+
module ActiveStorage
|
|
4
|
+
module AwsRecord
|
|
5
|
+
# Wires the gem into a Rails app: maps +config.activestorage_aws_record+ onto
|
|
6
|
+
# the gem's {Configuration}, registers the three classes as Active Storage's
|
|
7
|
+
# persistence classes before class indirection resolves, then — once the app
|
|
8
|
+
# is initialized — discovers the table layout and declares the models' key
|
|
9
|
+
# attributes, sets up the service registry (the custom blob class does not
|
|
10
|
+
# load the default Active Record blob that normally does this), and installs
|
|
11
|
+
# the +preview_image+ / +image+ attachments.
|
|
12
|
+
class Railtie < ::Rails::Railtie
|
|
13
|
+
config.activestorage_aws_record = ActiveSupport::OrderedOptions.new
|
|
14
|
+
|
|
15
|
+
initializer 'activestorage_aws_record.configure', before: 'active_storage.class_indirection' do |app|
|
|
16
|
+
options = app.config.activestorage_aws_record
|
|
17
|
+
|
|
18
|
+
ActiveStorage::AwsRecord.configure do |config|
|
|
19
|
+
config.table_name = options.table_name if options.table_name
|
|
20
|
+
config.namespace = options.namespace if options.namespace
|
|
21
|
+
config.separator = options.separator if options.separator
|
|
22
|
+
config.manage_table = options.manage_table unless options.manage_table.nil?
|
|
23
|
+
config.index_name = options.index_name if options.index_name
|
|
24
|
+
config.client_options = options.client_options if options.client_options
|
|
25
|
+
config.client = options.client if options.client
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
app.config.active_storage.blob_class = 'ActiveStorage::AwsRecord::Blob'
|
|
29
|
+
app.config.active_storage.attachment_class = 'ActiveStorage::AwsRecord::Attachment'
|
|
30
|
+
app.config.active_storage.variant_record_class = 'ActiveStorage::AwsRecord::VariantRecord'
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Run after the app is fully initialized so Active Storage's app/models
|
|
34
|
+
# (Servable, Variant, ...) are autoloadable when our model classes load.
|
|
35
|
+
config.after_initialize do |app|
|
|
36
|
+
# Discover the table layout and declare the models' key attributes +
|
|
37
|
+
# shared client.
|
|
38
|
+
ActiveStorage::AwsRecord.install!
|
|
39
|
+
|
|
40
|
+
# The default AR blob initializes the service registry on load; the
|
|
41
|
+
# custom backend must do it explicitly.
|
|
42
|
+
ActiveStorage::Services.setup_from_app_config(app)
|
|
43
|
+
|
|
44
|
+
# Install the owner attachments now that class indirection is configured.
|
|
45
|
+
ActiveStorage::AwsRecord.install_attachments!
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
rake_tasks do
|
|
49
|
+
load File.expand_path('tasks.rake', __dir__)
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
module ActiveStorage
|
|
2
|
+
module AwsRecord
|
|
3
|
+
# A lazy relation returned by +Attachment.where+ / +Attachment.find_by+. It
|
|
4
|
+
# mirrors the in-memory reference +Relation+: it collects filters/exclusions
|
|
5
|
+
# and, on materialization, runs the cheapest access path for the single-table
|
|
6
|
+
# layout — an owner-adjacency query — then applies residual filters,
|
|
7
|
+
# exclusions, and ordering in Ruby (the attachment set for one owner+name is
|
|
8
|
+
# small). Unsupported filters raise +ActiveStorage::QueryNotSupported+ rather
|
|
9
|
+
# than silently scanning.
|
|
10
|
+
#
|
|
11
|
+
# In Mode A the query is a strongly-consistent base-table query; in Mode B it
|
|
12
|
+
# runs against the string-keyed GSI (eventually consistent).
|
|
13
|
+
class Relation
|
|
14
|
+
include Enumerable
|
|
15
|
+
|
|
16
|
+
def initialize(model, filters: {}, exclusions: [])
|
|
17
|
+
@model = model
|
|
18
|
+
@filters = filters.transform_keys(&:to_sym)
|
|
19
|
+
@exclusions = exclusions
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def where(attributes = nil)
|
|
23
|
+
return WhereChain.new(self) if attributes.nil?
|
|
24
|
+
|
|
25
|
+
self.class.new(@model, filters: @filters.merge(attributes), exclusions: @exclusions)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def not(attributes)
|
|
29
|
+
self.class.new(@model, filters: @filters, exclusions: @exclusions + [attributes.transform_keys(&:to_sym)])
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def order(*attributes)
|
|
33
|
+
Ordered.new(to_a, attributes.flatten)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def find_by(attributes)
|
|
37
|
+
where(attributes).first
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def each(&block)
|
|
41
|
+
to_a.each(&block)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def pluck(*attrs)
|
|
45
|
+
to_a.map do |record|
|
|
46
|
+
values = attrs.map { |a| record.public_send(a) }
|
|
47
|
+
attrs.size == 1 ? values.first : values
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Detach-many's bulk delete. Wrapped in the model's transaction so the rows
|
|
52
|
+
# (and their coalesced blob refcount decrements) commit atomically instead
|
|
53
|
+
# of one-at-a-time, matching the contract's +attachment_class.transaction+
|
|
54
|
+
# expectation for grouped deletes.
|
|
55
|
+
def delete_all
|
|
56
|
+
records = to_a
|
|
57
|
+
@model.transaction { records.each(&:delete) }
|
|
58
|
+
records
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def to_a
|
|
62
|
+
@to_a ||= filtered(query_records)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def reload
|
|
66
|
+
@to_a = nil
|
|
67
|
+
self
|
|
68
|
+
end
|
|
69
|
+
alias_method :reset, :reload
|
|
70
|
+
|
|
71
|
+
# Backend filtering beyond the supported keys is not available on the
|
|
72
|
+
# generated collections; query the model class directly instead.
|
|
73
|
+
def method_missing(name, *, &)
|
|
74
|
+
raise ActiveStorage::QueryNotSupported,
|
|
75
|
+
"#{name} is not supported on #{self.class.name}; query #{@model.name}.where(...) directly."
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def respond_to_missing?(*)
|
|
79
|
+
false
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
private
|
|
83
|
+
|
|
84
|
+
# The contract always supplies (record_type, record_id[, name]); anything
|
|
85
|
+
# else has no single-table access path and must not silently scan.
|
|
86
|
+
def query_records
|
|
87
|
+
unless @filters.key?(:record_type) && @filters.key?(:record_id)
|
|
88
|
+
raise ActiveStorage::QueryNotSupported,
|
|
89
|
+
"Unsupported attachment query #{@filters.keys.inspect}; supported: (record_type, record_id[, name])."
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
owner_query
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def owner_query
|
|
96
|
+
schema = ActiveStorage::AwsRecord.schema
|
|
97
|
+
partition = @model.owner_partition(@filters[:record_type], @filters[:record_id])
|
|
98
|
+
prefix = @model.attachment_prefix(@filters[:name])
|
|
99
|
+
|
|
100
|
+
opts = {
|
|
101
|
+
key_condition_expression: '#h = :h AND begins_with(#r, :r)',
|
|
102
|
+
expression_attribute_values: { ':h' => partition, ':r' => prefix },
|
|
103
|
+
}
|
|
104
|
+
if schema.range_mode?
|
|
105
|
+
opts[:expression_attribute_names] = { '#h' => schema.partition_attr, '#r' => schema.sort_attr }
|
|
106
|
+
opts[:consistent_read] = true
|
|
107
|
+
else
|
|
108
|
+
opts[:index_name] = schema.index_name
|
|
109
|
+
opts[:expression_attribute_names] = { '#h' => schema.index_partition_attr, '#r' => schema.index_sort_attr }
|
|
110
|
+
end
|
|
111
|
+
@model.query(opts).to_a
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Apply residual equality filters and exclusions in Ruby.
|
|
115
|
+
def filtered(records)
|
|
116
|
+
records.select do |record|
|
|
117
|
+
@filters.all? { |attr, value| matches?(record.public_send(attr), value) } &&
|
|
118
|
+
@exclusions.none? { |excl| excl.any? { |attr, value| matches?(record.public_send(attr), value) } }
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def matches?(actual, expected)
|
|
123
|
+
if expected.is_a?(Array)
|
|
124
|
+
expected.map(&:to_s).include?(actual.to_s)
|
|
125
|
+
else
|
|
126
|
+
actual.to_s == expected.to_s
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# Materialized, ordered result that still supports the collection helpers.
|
|
131
|
+
class Ordered
|
|
132
|
+
include Enumerable
|
|
133
|
+
|
|
134
|
+
def initialize(records, attributes)
|
|
135
|
+
@records = records.sort_by { |record| attributes.map { |attr| sort_value(record.public_send(attr)) } }
|
|
136
|
+
@attributes = attributes
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def each(&block) = @records.each(&block)
|
|
140
|
+
def to_a = @records.dup
|
|
141
|
+
def pluck(*attrs)
|
|
142
|
+
@records.map { |r| attrs.size == 1 ? r.public_send(attrs.first) : attrs.map { |a| r.public_send(a) } }
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
private
|
|
146
|
+
|
|
147
|
+
def sort_value(value)
|
|
148
|
+
value.nil? ? '' : value.to_s
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
class WhereChain
|
|
153
|
+
def initialize(relation)
|
|
154
|
+
@relation = relation
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def not(attributes)
|
|
158
|
+
@relation.not(attributes)
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
end
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
module ActiveStorage
|
|
2
|
+
module AwsRecord
|
|
3
|
+
# The resolved physical layout of the single table, discovered once at boot
|
|
4
|
+
# from +describe_table+ (with optional config overrides) and cached read-only
|
|
5
|
+
# thereafter. Two modes, chosen by the range key's type:
|
|
6
|
+
#
|
|
7
|
+
# * +:range+ (range key is String "S") — the gem writes its +#+-composite
|
|
8
|
+
# adjacency keys straight into the table's (partition, sort) attributes. All
|
|
9
|
+
# reads are on the base table and strongly consistent. No GSI.
|
|
10
|
+
# * +:index+ (range key is Number "N") — composite strings cannot live in a
|
|
11
|
+
# numeric range key, so each item is keyed by a unique +item_id+ plus a
|
|
12
|
+
# constant +0+ range, and the adjacency keys move to a string-keyed GSI that
|
|
13
|
+
# serves listing queries (eventually consistent). Point lookups, the
|
|
14
|
+
# refcount, and the foreign-key guard stay on the base table (strong).
|
|
15
|
+
#
|
|
16
|
+
# The partition key must be String in both modes (it stores +item_id+/key
|
|
17
|
+
# strings); a numeric partition key is rejected.
|
|
18
|
+
class Schema
|
|
19
|
+
# DynamoDB attribute names the gem stores for its own logical attributes.
|
|
20
|
+
# Detected key attributes must not collide with these (see {.guard_names!}).
|
|
21
|
+
RESERVED_ATTRIBUTE_NAMES = %w[
|
|
22
|
+
as_id as_key as_filename as_content_type as_byte_size as_checksum
|
|
23
|
+
as_metadata as_service_name as_created_at as_attachments_count
|
|
24
|
+
as_record_type as_record_id as_name as_blob_id as_variation_digest
|
|
25
|
+
as_entity
|
|
26
|
+
].freeze
|
|
27
|
+
|
|
28
|
+
attr_reader :mode, :partition_attr, :sort_attr,
|
|
29
|
+
:index_name, :index_partition_attr, :index_sort_attr
|
|
30
|
+
|
|
31
|
+
# Inspect the live table and resolve the layout.
|
|
32
|
+
#
|
|
33
|
+
# @param client [Aws::DynamoDB::Client]
|
|
34
|
+
# @param config [Configuration]
|
|
35
|
+
# @return [Schema]
|
|
36
|
+
# @raise [ConfigurationError] when the table cannot be adapted.
|
|
37
|
+
def self.discover(client, config)
|
|
38
|
+
table = describe(client, config.table_name)
|
|
39
|
+
new(table, config)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def self.describe(client, table_name)
|
|
43
|
+
client.describe_table(table_name: table_name).table
|
|
44
|
+
rescue Aws::DynamoDB::Errors::ResourceNotFoundException
|
|
45
|
+
raise ConfigurationError, "DynamoDB table #{table_name.inspect} does not exist. " \
|
|
46
|
+
'Create it first, or set config.manage_table = true in development.'
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def initialize(table, config)
|
|
50
|
+
types = table.attribute_definitions.each_with_object({}) { |d, h| h[d.attribute_name] = d.attribute_type }
|
|
51
|
+
|
|
52
|
+
@partition_attr = key_name(table.key_schema, 'HASH')
|
|
53
|
+
@sort_attr = key_name(table.key_schema, 'RANGE')
|
|
54
|
+
|
|
55
|
+
validate!(types)
|
|
56
|
+
|
|
57
|
+
case types[@sort_attr]
|
|
58
|
+
when 'S'
|
|
59
|
+
@mode = :range
|
|
60
|
+
when 'N'
|
|
61
|
+
@mode = :index
|
|
62
|
+
resolve_index!(table, config, types)
|
|
63
|
+
else
|
|
64
|
+
raise ConfigurationError, "Range key #{@sort_attr.inspect} must be type String (S) or Number (N), " \
|
|
65
|
+
"got #{types[@sort_attr].inspect}."
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
self.class.guard_names!(self)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# @return [Boolean] true in Mode A (string range key, base-table adjacency).
|
|
72
|
+
def range_mode? = @mode == :range
|
|
73
|
+
|
|
74
|
+
# @return [Boolean] true in Mode B (numeric range key, GSI adjacency).
|
|
75
|
+
def index_mode? = @mode == :index
|
|
76
|
+
|
|
77
|
+
private
|
|
78
|
+
|
|
79
|
+
def key_name(key_schema, type)
|
|
80
|
+
key_schema.find { it.key_type == type }&.attribute_name
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def validate!(types)
|
|
84
|
+
raise ConfigurationError, 'Table has no partition key' unless @partition_attr
|
|
85
|
+
raise ConfigurationError, "Table #{@partition_attr.inspect} has no range key; a " \
|
|
86
|
+
'composite (partition + sort) key is required.' unless @sort_attr
|
|
87
|
+
|
|
88
|
+
partition_type = types[@partition_attr]
|
|
89
|
+
if partition_type && partition_type != 'S'
|
|
90
|
+
raise ConfigurationError, "Partition key #{@partition_attr.inspect} must be type String (S), " \
|
|
91
|
+
"got #{partition_type.inspect}."
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Mode B needs a string-keyed GSI carrying the adjacency keys. Locate it by
|
|
96
|
+
# the configured name and adopt its actual key attribute names. Absent on an
|
|
97
|
+
# app-managed table → tell the operator exactly what to add rather than
|
|
98
|
+
# silently mutating their indexes. The GSI must have a (String, String) key
|
|
99
|
+
# and project ALL.
|
|
100
|
+
def resolve_index!(table, config, types)
|
|
101
|
+
@index_name = config.index_name
|
|
102
|
+
gsi = (table.global_secondary_indexes || []).find { it.index_name == @index_name }
|
|
103
|
+
unless gsi
|
|
104
|
+
raise ConfigurationError, "Range key #{@sort_attr.inspect} is numeric (N), so a string-keyed " \
|
|
105
|
+
"GSI named #{@index_name.inspect} is required (a String partition + String sort key, " \
|
|
106
|
+
'projection ALL). Add it to the table, or set config.manage_table = true in development.'
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
@index_partition_attr = key_name(gsi.key_schema, 'HASH')
|
|
110
|
+
@index_sort_attr = key_name(gsi.key_schema, 'RANGE')
|
|
111
|
+
|
|
112
|
+
validate_index!(gsi, types)
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def validate_index!(gsi, types)
|
|
116
|
+
unless @index_partition_attr && @index_sort_attr
|
|
117
|
+
raise ConfigurationError, "GSI #{@index_name.inspect} must have both a partition and a sort key."
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
[@index_partition_attr, @index_sort_attr].each do |attr|
|
|
121
|
+
type = types[attr]
|
|
122
|
+
if type && type != 'S'
|
|
123
|
+
raise ConfigurationError, "GSI #{@index_name.inspect} key #{attr.inspect} must be type String (S), " \
|
|
124
|
+
"got #{type.inspect}."
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
projection = gsi.projection&.projection_type
|
|
129
|
+
if projection && projection != 'ALL'
|
|
130
|
+
raise ConfigurationError, "GSI #{@index_name.inspect} must project ALL attributes (got #{projection.inspect})."
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Reject a table whose key (or index key) attribute names collide with the
|
|
135
|
+
# gem's stored attributes — aws-record cannot map two attributes to one
|
|
136
|
+
# DynamoDB name.
|
|
137
|
+
def self.guard_names!(schema)
|
|
138
|
+
names = [schema.partition_attr, schema.sort_attr,
|
|
139
|
+
schema.index_partition_attr, schema.index_sort_attr,].compact
|
|
140
|
+
clash = names & RESERVED_ATTRIBUTE_NAMES
|
|
141
|
+
unless clash.empty?
|
|
142
|
+
raise ConfigurationError, "Key attribute name(s) #{clash.inspect} collide with attributes the gem " \
|
|
143
|
+
'stores. Use different key attribute names for this table.'
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
end
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
module ActiveStorage
|
|
2
|
+
module AwsRecord
|
|
3
|
+
# Creates / deletes the gem's single DynamoDB table. Intended for development
|
|
4
|
+
# and tests (gated by +config.manage_table+); production tables are managed by
|
|
5
|
+
# the application. The created table uses a String (partition, sort) key —
|
|
6
|
+
# i.e. Mode A — with on-demand (PAY_PER_REQUEST) billing and no GSI. Point the
|
|
7
|
+
# gem at an existing table (any key-attribute names; numeric range → Mode B)
|
|
8
|
+
# to integrate with an app that owns its schema.
|
|
9
|
+
module Tables
|
|
10
|
+
module_function
|
|
11
|
+
|
|
12
|
+
def client
|
|
13
|
+
ActiveStorage::AwsRecord.dynamodb_client
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def table_name
|
|
17
|
+
ActiveStorage::AwsRecord.config.table_name
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Partition / sort key attribute names for a gem-created (Mode A) table.
|
|
21
|
+
# Production tables are app-managed and may use any names (auto-detected).
|
|
22
|
+
def partition_key
|
|
23
|
+
'pk'
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def sort_key
|
|
27
|
+
'sk'
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Create the table if it does not already exist.
|
|
31
|
+
def ensure!
|
|
32
|
+
return if exist?
|
|
33
|
+
|
|
34
|
+
create!
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def exist?
|
|
38
|
+
client.describe_table(table_name: table_name)
|
|
39
|
+
true
|
|
40
|
+
rescue Aws::DynamoDB::Errors::ResourceNotFoundException
|
|
41
|
+
false
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def create!
|
|
45
|
+
client.create_table(
|
|
46
|
+
table_name: table_name,
|
|
47
|
+
attribute_definitions: [
|
|
48
|
+
{ attribute_name: partition_key, attribute_type: 'S' },
|
|
49
|
+
{ attribute_name: sort_key, attribute_type: 'S' },
|
|
50
|
+
],
|
|
51
|
+
key_schema: [
|
|
52
|
+
{ attribute_name: partition_key, key_type: 'HASH' },
|
|
53
|
+
{ attribute_name: sort_key, key_type: 'RANGE' },
|
|
54
|
+
],
|
|
55
|
+
billing_mode: 'PAY_PER_REQUEST'
|
|
56
|
+
)
|
|
57
|
+
client.wait_until(:table_exists, table_name: table_name)
|
|
58
|
+
rescue Aws::DynamoDB::Errors::ResourceInUseException
|
|
59
|
+
nil
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def delete!
|
|
63
|
+
client.delete_table(table_name: table_name)
|
|
64
|
+
wait_until_gone!
|
|
65
|
+
rescue Aws::DynamoDB::Errors::ResourceNotFoundException
|
|
66
|
+
nil
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Drop and recreate (used by the test harness for a clean slate).
|
|
70
|
+
def reset!
|
|
71
|
+
delete!
|
|
72
|
+
create!
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def wait_until_gone!
|
|
76
|
+
client.wait_until(:table_not_exists, table_name: table_name)
|
|
77
|
+
rescue Aws::Waiters::Errors::WaiterFailed
|
|
78
|
+
nil
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
namespace :activestorage_aws_record do
|
|
2
|
+
namespace :table do
|
|
3
|
+
desc 'Create the Active Storage DynamoDB table'
|
|
4
|
+
task create: :environment do
|
|
5
|
+
ActiveStorage::AwsRecord::Tables.create!
|
|
6
|
+
puts "Created/verified table: #{ActiveStorage::AwsRecord.config.table_name}"
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
desc 'Delete the Active Storage DynamoDB table'
|
|
10
|
+
task delete: :environment do
|
|
11
|
+
ActiveStorage::AwsRecord::Tables.delete!
|
|
12
|
+
puts 'Deleted Active Storage DynamoDB table.'
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
module ActiveStorage
|
|
2
|
+
module AwsRecord
|
|
3
|
+
# Raised, before any write, when a batched attachment change would exceed
|
|
4
|
+
# DynamoDB's hard 100-action limit for +transact_write_items+. The operation
|
|
5
|
+
# fails closed rather than chunking or falling back to per-row deletes — both
|
|
6
|
+
# of which would reintroduce the partial-clear bug this batching exists to
|
|
7
|
+
# prevent. Split the change into smaller batches instead.
|
|
8
|
+
class TransactionTooLarge < StandardError; end
|
|
9
|
+
|
|
10
|
+
# A fiber-local accumulator that batches the attachment *destroys* opened
|
|
11
|
+
# inside an +Attachment.transaction+ block into a single DynamoDB
|
|
12
|
+
# +transact_write_items+, so a multi-attachment clear / replace / detach is
|
|
13
|
+
# atomic: DynamoDB deletes every row (and adjusts every blob refcount) or
|
|
14
|
+
# none, instead of deleting some rows before a later one fails (the
|
|
15
|
+
# partial-clear bug the generic +has_many+ paths would otherwise hit).
|
|
16
|
+
#
|
|
17
|
+
# *Creates are intentionally not buffered.* Active Storage's own create paths
|
|
18
|
+
# (+CreateOne+/+CreateMany+) clean up newly-built blob/attachment records when
|
|
19
|
+
# a +save!+ raises synchronously; deferring those writes to commit would move
|
|
20
|
+
# the failure past that cleanup. Only destroys — whose sole failure mode is
|
|
21
|
+
# *partial* multi-row deletion — are batched here.
|
|
22
|
+
#
|
|
23
|
+
# Fiber-safe (Falcon): the active context lives in +Fiber[]+, never a class or
|
|
24
|
+
# thread global, so independent request fibers (Falcon assigns one per request)
|
|
25
|
+
# each get their own transaction. The grouped-destroy blocks are executed
|
|
26
|
+
# synchronously on one fiber; +Fiber[]+ is inherited by a *child* fiber, so do
|
|
27
|
+
# not spawn an Async task / +Fiber.new+ inside an +Attachment.transaction+ block
|
|
28
|
+
# (it would join the parent's buffer rather than open its own).
|
|
29
|
+
class Transaction
|
|
30
|
+
# DynamoDB's hard ceiling on actions in one +transact_write_items+ call.
|
|
31
|
+
MAX_TRANSACT_ITEMS = 100
|
|
32
|
+
|
|
33
|
+
FIBER_KEY = :active_storage_aws_record_transaction
|
|
34
|
+
|
|
35
|
+
class << self
|
|
36
|
+
# @return [Transaction, nil] the transaction active on this fiber.
|
|
37
|
+
def current
|
|
38
|
+
Fiber[FIBER_KEY]
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Run +block+ inside a transaction. A *nested* call joins the enclosing
|
|
42
|
+
# transaction — its buffered destroys flush only at the outermost commit.
|
|
43
|
+
# The outermost call commits the buffer on success; on any exception it
|
|
44
|
+
# discards the buffer *unwritten*, so the block is atomic (nothing is
|
|
45
|
+
# deleted). The fiber-local context is always cleared on the way out.
|
|
46
|
+
#
|
|
47
|
+
# @param model [Class] the Attachment class that opened the transaction.
|
|
48
|
+
def run(model)
|
|
49
|
+
return yield if current # nested: join the outer transaction
|
|
50
|
+
|
|
51
|
+
tx = new(model)
|
|
52
|
+
Fiber[FIBER_KEY] = tx
|
|
53
|
+
begin
|
|
54
|
+
result = yield
|
|
55
|
+
tx.commit!
|
|
56
|
+
result
|
|
57
|
+
ensure
|
|
58
|
+
Fiber[FIBER_KEY] = nil
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def initialize(model)
|
|
64
|
+
@model = model
|
|
65
|
+
@destroys = []
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Buffer one attachment destroy/delete for the commit.
|
|
69
|
+
def enqueue_destroy(attachment)
|
|
70
|
+
@destroys << attachment
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Flush the buffered destroys:
|
|
74
|
+
# * 0 → nothing;
|
|
75
|
+
# * 1 → the existing per-row path (+#commit_destroy!+), which keeps its
|
|
76
|
+
# idempotent duplicate-purge / orphaned-blob recovery;
|
|
77
|
+
# * ≥2 → one coalesced +transact_write_items+ (one delete per attachment row,
|
|
78
|
+
# one refcount +ADD+ per *distinct* blob, since DynamoDB forbids two
|
|
79
|
+
# actions on the same item in a transaction).
|
|
80
|
+
def commit!
|
|
81
|
+
return if @destroys.empty?
|
|
82
|
+
|
|
83
|
+
if @destroys.size == 1
|
|
84
|
+
@destroys.first.commit_destroy!
|
|
85
|
+
return
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
items = transact_items
|
|
89
|
+
if items.size > MAX_TRANSACT_ITEMS
|
|
90
|
+
deletes = items.count { |item| item.key?(:delete) }
|
|
91
|
+
raise TransactionTooLarge,
|
|
92
|
+
"Atomic attachment change needs #{items.size} DynamoDB actions, over the " \
|
|
93
|
+
"#{MAX_TRANSACT_ITEMS}-action transaction limit (#{deletes} attachments, " \
|
|
94
|
+
"#{items.size - deletes} blobs). Split the change into smaller batches."
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
@model.dynamodb_client.transact_write_items(transact_items: items)
|
|
98
|
+
@destroys.each(&:mark_destroyed!)
|
|
99
|
+
rescue Aws::DynamoDB::Errors::TransactionCanceledException => e
|
|
100
|
+
# A conditional failure cancels the *whole* batch (a row or blob vanished
|
|
101
|
+
# under a concurrent change), so nothing was written. Surface it as a
|
|
102
|
+
# destroy failure; the generic path resets its deferred purges and
|
|
103
|
+
# re-raises. Refusing a partial result here is the point of batching.
|
|
104
|
+
raise ActiveStorage::RecordNotDestroyed.new(
|
|
105
|
+
"Failed to destroy #{@destroys.size} attachments atomically: #{e.message}", @destroys.first
|
|
106
|
+
)
|
|
107
|
+
rescue Aws::DynamoDB::Errors::ServiceError => e
|
|
108
|
+
# Any other DynamoDB failure (throttling, network) also wrote nothing;
|
|
109
|
+
# map it to RecordNotDestroyed too, matching the per-row #destroy contract.
|
|
110
|
+
raise ActiveStorage::RecordNotDestroyed.new("Failed to destroy attachment: #{e.message}", @destroys.first)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
private
|
|
114
|
+
|
|
115
|
+
# One delete per attachment row, plus one coalesced refcount update per
|
|
116
|
+
# distinct blob (delta = −the number of its buffered destroys), so the same
|
|
117
|
+
# blob attached twice yields a single +ADD #c -2+ rather than two rejected
|
|
118
|
+
# operations on one item. Rows are de-duplicated by physical key first: the
|
|
119
|
+
# generic paths only ever enqueue distinct rows, but the public reentrant
|
|
120
|
+
# +Attachment.transaction+ could buffer the *same* row twice (e.g. a nested
|
|
121
|
+
# purge), which DynamoDB would reject as two actions on one item — collapsing
|
|
122
|
+
# it to a single delete keeps that an idempotent no-op, as the per-row path is.
|
|
123
|
+
def transact_items
|
|
124
|
+
rows = @destroys.uniq(&:physical_key)
|
|
125
|
+
deletes = rows.map { |attachment| { delete: attachment.delete_transact_item } }
|
|
126
|
+
deltas = rows.each_with_object(Hash.new(0)) { |attachment, h| h[attachment.blob_id] -= 1 }
|
|
127
|
+
updates = deltas.map { |blob_id, delta| { update: @model.blob_count_update(blob_id, delta) } }
|
|
128
|
+
deletes + updates
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|