tape_deck 0.1.0.beta1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.devcontainer/compose.yml +18 -2
- data/CHANGELOG.md +18 -0
- data/DEVELOPMENT.md +29 -18
- data/README.md +79 -14
- data/Rakefile +50 -15
- data/lib/tape_deck/adapters/base.rb +96 -0
- data/lib/tape_deck/adapters/mysql.rb +20 -0
- data/lib/tape_deck/adapters/postgres.rb +29 -0
- data/lib/tape_deck/adapters/sqlite.rb +10 -0
- data/lib/tape_deck/adapters.rb +22 -0
- data/lib/tape_deck/configuration.rb +116 -3
- data/lib/tape_deck/filter.rb +1 -2
- data/lib/tape_deck/new_rows.rb +168 -0
- data/lib/tape_deck/recorder.rb +97 -90
- data/lib/tape_deck/rows.rb +16 -0
- data/lib/tape_deck/session.rb +73 -30
- data/lib/tape_deck/tape.rb +47 -34
- data/lib/tape_deck/version.rb +1 -1
- data/lib/tape_deck.rb +65 -8
- metadata +10 -3
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "adapters/base"
|
|
4
|
+
require_relative "adapters/mysql"
|
|
5
|
+
require_relative "adapters/postgres"
|
|
6
|
+
require_relative "adapters/sqlite"
|
|
7
|
+
|
|
8
|
+
module TapeDeck
|
|
9
|
+
# The database specific parts of taking snapshots: telling which tables changed, and turning column values into
|
|
10
|
+
# plain YAML values. Everything else goes through Sequel and works the same for every database.
|
|
11
|
+
module Adapters
|
|
12
|
+
# Adapters by +Sequel::Database#database_type+.
|
|
13
|
+
BY_DATABASE_TYPE = {mysql: MySQL, postgres: Postgres, sqlite: SQLite}.freeze
|
|
14
|
+
|
|
15
|
+
# @param db [Sequel::Database]
|
|
16
|
+
# @return [Base] the adapter for the database type. {Base} for other databases, which works, but reads every
|
|
17
|
+
# table on every snapshot.
|
|
18
|
+
def self.for(db)
|
|
19
|
+
BY_DATABASE_TYPE.fetch(db.database_type, Base).new(db)
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -33,6 +33,21 @@ module TapeDeck
|
|
|
33
33
|
# @return [Array<String>] patterns added with {#exclude_columns}
|
|
34
34
|
attr_reader :excluded_columns
|
|
35
35
|
|
|
36
|
+
# @return [Array<String>] patterns added with {#include_views}
|
|
37
|
+
attr_reader :included_views
|
|
38
|
+
|
|
39
|
+
# @return [Array<String>] patterns added with {#keep_new_ids}
|
|
40
|
+
attr_reader :kept_new_ids
|
|
41
|
+
|
|
42
|
+
# @return [Array<String>] patterns added with {#number_by_content}
|
|
43
|
+
attr_reader :numbered_by_content
|
|
44
|
+
|
|
45
|
+
# @return [Array<String>] patterns added with {#generated_keys}
|
|
46
|
+
attr_reader :generated_key_columns
|
|
47
|
+
|
|
48
|
+
# @return [Hash{String => Array<String>}] patterns added with {#references_to}, by referenced table
|
|
49
|
+
attr_reader :reference_columns
|
|
50
|
+
|
|
36
51
|
def initialize
|
|
37
52
|
@tape_library_dir = "spec/db_tapes"
|
|
38
53
|
@databases = {}
|
|
@@ -40,6 +55,11 @@ module TapeDeck
|
|
|
40
55
|
@on_schema_change = :error
|
|
41
56
|
@excluded_tables = []
|
|
42
57
|
@excluded_columns = []
|
|
58
|
+
@included_views = []
|
|
59
|
+
@kept_new_ids = []
|
|
60
|
+
@numbered_by_content = []
|
|
61
|
+
@generated_key_columns = []
|
|
62
|
+
@reference_columns = {}
|
|
43
63
|
end
|
|
44
64
|
|
|
45
65
|
# @param mode [Symbol] one of {Tape::RECORD_MODES}
|
|
@@ -80,9 +100,72 @@ module TapeDeck
|
|
|
80
100
|
@excluded_columns.concat(patterns.map(&:to_s))
|
|
81
101
|
end
|
|
82
102
|
|
|
103
|
+
# Adds matching views to snapshots. Views are left out by default, because the changes of the tables behind them
|
|
104
|
+
# are recorded anyway. Include them to also notice changed view definitions, or data a view reads from tables
|
|
105
|
+
# that are not recorded. Included views are read in full on every snapshot, and compared as a multiset of rows,
|
|
106
|
+
# because they have no primary key. {#exclude_columns} applies to their columns, {#exclude_tables} does not.
|
|
107
|
+
#
|
|
108
|
+
# @param patterns [Array<String>] globs on +"<database>.<view>"+
|
|
109
|
+
# @return [void]
|
|
110
|
+
# @example
|
|
111
|
+
# c.include_views "shop.order_totals", "billing.*"
|
|
112
|
+
def include_views(*patterns)
|
|
113
|
+
@included_views.concat(patterns.map(&:to_s))
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Keeps the generated ids of rows inserted into matching tables, instead of replacing them with placeholders.
|
|
117
|
+
# See "New rows" in {TapeDeck}.
|
|
118
|
+
#
|
|
119
|
+
# @param patterns [Array<String>] globs on +"<database>.<table>"+
|
|
120
|
+
# @return [void]
|
|
121
|
+
# @example
|
|
122
|
+
# c.keep_new_ids "shop.audit_log"
|
|
123
|
+
def keep_new_ids(*patterns)
|
|
124
|
+
@kept_new_ids.concat(patterns.map(&:to_s))
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# Numbers the new rows of matching tables by their content instead of the order they were created in, so creating
|
|
128
|
+
# the same rows in a different order still verifies. For example an import, whose order doesn't matter. Rows with
|
|
129
|
+
# the same content are numbered in creation order. See "New rows" in {TapeDeck}.
|
|
130
|
+
#
|
|
131
|
+
# @param patterns [Array<String>] globs on +"<database>.<table>"+
|
|
132
|
+
# @return [void]
|
|
133
|
+
# @example
|
|
134
|
+
# c.number_by_content "shop.imported_products"
|
|
135
|
+
def number_by_content(*patterns)
|
|
136
|
+
@numbered_by_content.concat(patterns.map(&:to_s))
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# Marks matching primary keys as generated, like auto increment ones are: keys that the database or the
|
|
140
|
+
# application fills in, for example UUIDs. Their new rows get placeholders, see "New rows" in {TapeDeck}. Keys that
|
|
141
|
+
# are not integers have no creation order, so their rows are numbered by content.
|
|
142
|
+
#
|
|
143
|
+
# @param patterns [Array<String>] globs on +"<database>.<table>.<column>"+
|
|
144
|
+
# @return [void]
|
|
145
|
+
# @example
|
|
146
|
+
# c.generated_keys "shop.tokens.id", "*.uuid"
|
|
147
|
+
def generated_keys(*patterns)
|
|
148
|
+
@generated_key_columns.concat(patterns.map(&:to_s))
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# Marks matching columns as holding keys of a table, like a declared foreign key does. Needed for references
|
|
152
|
+
# across databases, and ones without a foreign key. Overrides a declared foreign key of the same column.
|
|
153
|
+
#
|
|
154
|
+
# @param table [String] +"<database>.<table>"+ whose keys the columns hold
|
|
155
|
+
# @param patterns [Array<String>] globs on +"<database>.<table>.<column>"+
|
|
156
|
+
# @return [void]
|
|
157
|
+
# @example
|
|
158
|
+
# c.references_to "shop.orders", "billing.invoices.order_id", "*.last_order_id"
|
|
159
|
+
def references_to(table, *patterns)
|
|
160
|
+
(@reference_columns[table.to_s] ||= []).concat(patterns.map(&:to_s))
|
|
161
|
+
end
|
|
162
|
+
|
|
83
163
|
# Wraps RSpec examples tagged +:tape_deck+ in a tape. The tape is named after the example's description hierarchy
|
|
84
164
|
# unless +tape_name:+ is given. When the example fails, the tape is neither recorded nor verified.
|
|
85
165
|
#
|
|
166
|
+
# Before the first example runs, the suite fails if tagged examples would share a tape file, for example the same
|
|
167
|
+
# description in a loop. See {.check_rspec_tape_names}.
|
|
168
|
+
#
|
|
86
169
|
# @return [void]
|
|
87
170
|
# @example
|
|
88
171
|
# it "creates an order", :tape_deck do ... end
|
|
@@ -90,11 +173,13 @@ module TapeDeck
|
|
|
90
173
|
# it "creates an order", tape_deck: { databases: %w[shop] } do ... end
|
|
91
174
|
def configure_rspec_metadata!
|
|
92
175
|
::RSpec.configure do |config|
|
|
176
|
+
# All examples are loaded by now, so examples sharing a tape by mistake fail the run before anything is recorded
|
|
177
|
+
config.before(:suite) { Configuration.check_rspec_tape_names(::RSpec.world.all_examples) }
|
|
178
|
+
|
|
93
179
|
config.around(:each, :tape_deck) do |example|
|
|
94
|
-
options =
|
|
95
|
-
name = options[:tape_name] || Configuration.rspec_tape_name(example)
|
|
180
|
+
name, options = Configuration.rspec_tape(example)
|
|
96
181
|
|
|
97
|
-
TapeDeck.insert_tape(name, **options
|
|
182
|
+
TapeDeck.insert_tape(name, **options)
|
|
98
183
|
example.run
|
|
99
184
|
example_failed = !example.exception.nil?
|
|
100
185
|
TapeDeck.eject_tape(discard: example_failed)
|
|
@@ -102,6 +187,34 @@ module TapeDeck
|
|
|
102
187
|
end
|
|
103
188
|
end
|
|
104
189
|
|
|
190
|
+
# @param example [RSpec::Core::Example] an example tagged +:tape_deck+
|
|
191
|
+
# @return [Array(String, Hash)] its tape name, and the options for {TapeDeck.insert_tape}
|
|
192
|
+
# @api private
|
|
193
|
+
def self.rspec_tape(example)
|
|
194
|
+
options = Hash.try_convert(example.metadata[:tape_deck]) || {} # a bare :tape_deck tag is `true`
|
|
195
|
+
name = options[:tape_name] || rspec_tape_name(example)
|
|
196
|
+
[name, options.except(:tape_name)]
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# Examples that share a tape file would record and verify each other's changes, for example the same description
|
|
200
|
+
# in a loop. Explicit +tape_name:+ options count too.
|
|
201
|
+
#
|
|
202
|
+
# @param examples [Array<RSpec::Core::Example>] all loaded examples
|
|
203
|
+
# @return [void]
|
|
204
|
+
# @raise [Error] if tagged examples share a tape file, listing them
|
|
205
|
+
# @api private
|
|
206
|
+
def self.check_rspec_tape_names(examples)
|
|
207
|
+
tagged = examples.select { _1.metadata[:tape_deck] }
|
|
208
|
+
shared = tagged.group_by { Tape.file_name_for(rspec_tape(_1).first) }.select { |_, group| group.size > 1 }
|
|
209
|
+
return if shared.empty?
|
|
210
|
+
|
|
211
|
+
details = shared.map do |file_name, group|
|
|
212
|
+
lines = group.map { " #{_1.id} #{rspec_tape(_1).first.inspect}" }
|
|
213
|
+
" #{file_name}:\n#{lines.join("\n")}"
|
|
214
|
+
end
|
|
215
|
+
raise Error, "TapeDeck: examples share a tape. Give each its own description or tape_name:\n#{details.join("\n")}"
|
|
216
|
+
end
|
|
217
|
+
|
|
105
218
|
# @param example [RSpec::Core::Example]
|
|
106
219
|
# @return [String] the example's description hierarchy, e.g. +"Checkout/creates an order"+
|
|
107
220
|
# @api private
|
data/lib/tape_deck/filter.rb
CHANGED
|
@@ -43,10 +43,9 @@ module TapeDeck
|
|
|
43
43
|
kept.reject { |_, entries| entries.empty? }
|
|
44
44
|
end
|
|
45
45
|
|
|
46
|
-
# Rows are sorted again, because their recorded order may depend on values that are now ignored.
|
|
47
46
|
def filter_rows(table_name, rows)
|
|
48
47
|
kept = rows.map { |row| filter_columns(table_name, row) }
|
|
49
|
-
kept.reject(&:empty?)
|
|
48
|
+
kept.reject(&:empty?)
|
|
50
49
|
end
|
|
51
50
|
|
|
52
51
|
# The "key" of an update is kept as it is, because it identifies the row.
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module TapeDeck
|
|
4
|
+
# Replaces the generated keys of inserted rows with placeholders like +"new shop.orders 1"+, numbered per table, and
|
|
5
|
+
# references to those rows with the same placeholder. So a tape doesn't depend on where the database's counters
|
|
6
|
+
# stand. Ids of rows that existed before stay as they are. See "New rows" in {TapeDeck}.
|
|
7
|
+
#
|
|
8
|
+
# @api private
|
|
9
|
+
class NewRows
|
|
10
|
+
# @param recorder [Recorder] tells the generated keys and references of tables
|
|
11
|
+
# @param keep [Array<String>] patterns on +"<database>.<table>"+ for tables whose new ids are kept, see
|
|
12
|
+
# {Configuration#keep_new_ids}
|
|
13
|
+
# @param by_content [Array<String>] patterns on +"<database>.<table>"+ for tables whose new rows are numbered by
|
|
14
|
+
# content instead of creation order, see {Configuration#number_by_content}
|
|
15
|
+
# @param generated_keys [Array<String>] patterns for more generated keys, see {Configuration#generated_keys}
|
|
16
|
+
# @param references [Hash{String => Array<String>}] more references, see {Configuration#references_to}
|
|
17
|
+
def initialize(recorder, keep: [], by_content: [], generated_keys: [], references: {})
|
|
18
|
+
@recorder = recorder
|
|
19
|
+
@keep = keep
|
|
20
|
+
@by_content = by_content
|
|
21
|
+
@generated_keys = generated_keys
|
|
22
|
+
@references = references
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# @param recorder [Recorder]
|
|
26
|
+
# @param tape_deck [Hash] the +tape_deck+ section of a tape, see {Tape#read}
|
|
27
|
+
# @return [NewRows] numbering with the rules the tape was recorded with
|
|
28
|
+
def self.recorded_with(recorder, tape_deck)
|
|
29
|
+
new(
|
|
30
|
+
recorder,
|
|
31
|
+
keep: tape_deck.fetch("keep_new_ids"),
|
|
32
|
+
by_content: tape_deck.fetch("number_by_content"),
|
|
33
|
+
generated_keys: tape_deck.fetch("generated_keys"),
|
|
34
|
+
references: tape_deck.fetch("references_to")
|
|
35
|
+
)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# @return [Hash] the rules, as stored in the +tape_deck+ section of a tape
|
|
39
|
+
def rules
|
|
40
|
+
{
|
|
41
|
+
"keep_new_ids" => @keep,
|
|
42
|
+
"number_by_content" => @by_content,
|
|
43
|
+
"generated_keys" => @generated_keys,
|
|
44
|
+
"references_to" => @references
|
|
45
|
+
}
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# @param diff [Hash] a diff as stored on a tape
|
|
49
|
+
# @return [Hash] the diff with placeholders
|
|
50
|
+
def apply(diff)
|
|
51
|
+
keys = @recorder.generated_keys(diff, patterns: @generated_keys).to_h do |db_name, tables|
|
|
52
|
+
[db_name, tables.reject { |table, _| Patterns.match?(@keep, "#{db_name}.#{table}") }]
|
|
53
|
+
end
|
|
54
|
+
references = @recorder.references(diff, patterns: @references)
|
|
55
|
+
columns = diff.to_h do |db_name, tables|
|
|
56
|
+
[db_name, tables.keys.to_h { [_1, placeholder_columns(db_name, _1, keys, references)] }]
|
|
57
|
+
end
|
|
58
|
+
placeholders = number(diff, keys, columns)
|
|
59
|
+
|
|
60
|
+
diff.to_h do |db_name, tables|
|
|
61
|
+
[db_name, tables.to_h { |table, changes| [table, replace(changes, columns.dig(db_name, table), placeholders)] }]
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
private
|
|
66
|
+
|
|
67
|
+
# @return [Hash{String => Array(String, String)}] column -> +[database, table]+ whose new keys it can hold: the
|
|
68
|
+
# table's own key, and its references
|
|
69
|
+
def placeholder_columns(db_name, table, keys, references)
|
|
70
|
+
key = keys.dig(db_name, table)
|
|
71
|
+
own_key = if key
|
|
72
|
+
{key => [db_name, table]}
|
|
73
|
+
else
|
|
74
|
+
{}
|
|
75
|
+
end
|
|
76
|
+
references.dig(db_name, table).to_h.merge(own_key)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Tables numbered by content go after the ones they reference, so their content can hold those placeholders.
|
|
80
|
+
# Keys that are not integers, like UUIDs, tell nothing about creation order, so their tables are numbered by
|
|
81
|
+
# content too.
|
|
82
|
+
#
|
|
83
|
+
# @return [Hash{Array(String, String) => Hash{Object => String}}] +[database, table]+ -> key -> placeholder
|
|
84
|
+
def number(diff, keys, columns)
|
|
85
|
+
tables = keys.flat_map { |db_name, db_keys| db_keys.map { |table, key| [db_name, table, key] } }
|
|
86
|
+
new_keys = tables.to_h do |db_name, table, key|
|
|
87
|
+
[[db_name, table], diff.dig(db_name, table, "inserted").map { _1.fetch(key) }]
|
|
88
|
+
end
|
|
89
|
+
by_content, by_creation = tables.partition do |db_name, table, _|
|
|
90
|
+
content?(db_name, table) || !new_keys.fetch([db_name, table]).all?(Integer)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
placeholders = {}
|
|
94
|
+
by_creation.each do |db_name, table, key|
|
|
95
|
+
rows = diff.dig(db_name, table, "inserted").sort_by { _1.fetch(key) }
|
|
96
|
+
placeholders[[db_name, table]] = assign(db_name, table, rows.map { _1.fetch(key) })
|
|
97
|
+
end
|
|
98
|
+
pending = by_content
|
|
99
|
+
until pending.empty?
|
|
100
|
+
ready = pending.reject do |db_name, table, _|
|
|
101
|
+
references_pending?(columns.dig(db_name, table), [db_name, table], pending)
|
|
102
|
+
end
|
|
103
|
+
ready = pending if ready.empty? # a cycle, see #content
|
|
104
|
+
ready.each do |db_name, table, key|
|
|
105
|
+
rows = diff.dig(db_name, table, "inserted")
|
|
106
|
+
sorted = rows.sort_by { [content(_1, key, columns.dig(db_name, table), placeholders, new_keys), _1.fetch(key)] }
|
|
107
|
+
placeholders[[db_name, table]] = assign(db_name, table, sorted.map { _1.fetch(key) })
|
|
108
|
+
end
|
|
109
|
+
pending -= ready
|
|
110
|
+
end
|
|
111
|
+
placeholders
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def content?(db_name, table) = Patterns.match?(@by_content, "#{db_name}.#{table}")
|
|
115
|
+
|
|
116
|
+
def references_pending?(columns, own, pending)
|
|
117
|
+
pending_tables = pending.map { |db_name, table, _| [db_name, table] } - [own]
|
|
118
|
+
columns.values.any? { pending_tables.include?(_1) }
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# The row without its key, as JSON. A reference to a new row that has no placeholder yet (the table references
|
|
122
|
+
# itself, or tables reference each other) only says which table it points to, because its id is not stable.
|
|
123
|
+
def content(row, key, columns, placeholders, new_keys)
|
|
124
|
+
values = row.except(key).to_h do |column, value|
|
|
125
|
+
target = columns[column]
|
|
126
|
+
replaced = if target.nil?
|
|
127
|
+
value
|
|
128
|
+
elsif placeholders.key?(target)
|
|
129
|
+
placeholders.dig(target, value) || value
|
|
130
|
+
elsif new_keys.fetch(target, []).include?(value)
|
|
131
|
+
"new #{target.join(".")}"
|
|
132
|
+
else
|
|
133
|
+
value
|
|
134
|
+
end
|
|
135
|
+
[column, replaced]
|
|
136
|
+
end
|
|
137
|
+
JSON.generate(values, allow_nan: true)
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def assign(db_name, table, values)
|
|
141
|
+
values.each_with_index.to_h { |value, i| [value, "new #{db_name}.#{table} #{i + 1}"] }
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# Rows that existed before keep their own ids, so only inserted rows and updated values can hold new ones.
|
|
145
|
+
#
|
|
146
|
+
# @param columns [Hash{String => Array(String, String)}] column -> +[database, table]+ whose new keys it can hold
|
|
147
|
+
def replace(changes, columns, placeholders)
|
|
148
|
+
swap = ->(column, value) do
|
|
149
|
+
target = columns[column]
|
|
150
|
+
(target && placeholders.dig(target, value)) || value
|
|
151
|
+
end
|
|
152
|
+
changes.to_h do |section, entries|
|
|
153
|
+
replaced = case section
|
|
154
|
+
when "inserted"
|
|
155
|
+
entries.map { |row| row.to_h { |column, value| [column, swap.call(column, value)] } }
|
|
156
|
+
when "updated"
|
|
157
|
+
entries.map do |update|
|
|
158
|
+
values = update.fetch("changes").to_h { |column, pair| [column, pair.map { swap.call(column, _1) }] }
|
|
159
|
+
update.merge("changes" => values)
|
|
160
|
+
end
|
|
161
|
+
else
|
|
162
|
+
entries
|
|
163
|
+
end
|
|
164
|
+
[section, replaced]
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
end
|
data/lib/tape_deck/recorder.rb
CHANGED
|
@@ -4,7 +4,7 @@ module TapeDeck
|
|
|
4
4
|
# Snapshot of one table.
|
|
5
5
|
#
|
|
6
6
|
# @!attribute [r] checksum
|
|
7
|
-
# @return [
|
|
7
|
+
# @return [Object, nil] checksum from the adapter, see {Adapters::Base#checksums}
|
|
8
8
|
# @!attribute [r] pk
|
|
9
9
|
# @return [Array<String>] primary key columns, empty if the table has none
|
|
10
10
|
# @!attribute [r] rows
|
|
@@ -13,26 +13,28 @@ module TapeDeck
|
|
|
13
13
|
# @api private
|
|
14
14
|
TableSnap = Data.define(:checksum, :pk, :rows)
|
|
15
15
|
|
|
16
|
-
# Takes snapshots of all configured databases and diffs them.
|
|
17
|
-
# need tape files.
|
|
16
|
+
# Takes snapshots of all configured databases and diffs them. The database specific parts are in {Adapters}. Used by
|
|
17
|
+
# {Session}, but usable on its own when you don't need tape files.
|
|
18
18
|
class Recorder
|
|
19
|
-
# Database types of TIME columns, with or without fractional seconds. Sequel only types +time+ as +:time+, not
|
|
20
|
-
# +time(6)+.
|
|
21
|
-
TIME_DB_TYPE = /\Atime\b/i
|
|
22
|
-
|
|
23
19
|
# @return [Array<String>] see {Configuration#exclude_tables}
|
|
24
20
|
attr_reader :excluded_tables
|
|
25
21
|
|
|
26
22
|
# @return [Array<String>] see {Configuration#exclude_columns}
|
|
27
23
|
attr_reader :excluded_columns
|
|
28
24
|
|
|
25
|
+
# @return [Array<String>] see {Configuration#include_views}
|
|
26
|
+
attr_reader :included_views
|
|
27
|
+
|
|
29
28
|
# @param dbs [Hash{String => Sequel::Database}]
|
|
30
29
|
# @param exclude_tables [Array<String>] see {Configuration#exclude_tables}
|
|
31
30
|
# @param exclude_columns [Array<String>] see {Configuration#exclude_columns}
|
|
32
|
-
|
|
31
|
+
# @param include_views [Array<String>] see {Configuration#include_views}
|
|
32
|
+
def initialize(dbs, exclude_tables: [], exclude_columns: [], include_views: [])
|
|
33
33
|
@dbs = dbs
|
|
34
|
+
@adapters = dbs.transform_values { Adapters.for(_1) }
|
|
34
35
|
@excluded_tables = exclude_tables
|
|
35
36
|
@excluded_columns = exclude_columns
|
|
37
|
+
@included_views = include_views
|
|
36
38
|
end
|
|
37
39
|
|
|
38
40
|
# @yield the action whose database changes are recorded
|
|
@@ -43,8 +45,9 @@ module TapeDeck
|
|
|
43
45
|
diff(before, snapshot(previous: before))
|
|
44
46
|
end
|
|
45
47
|
|
|
46
|
-
# Reads all tables that are not excluded. With +previous:+, tables whose
|
|
47
|
-
# previous snapshot, so only changed tables are read again.
|
|
48
|
+
# Reads all tables that are not excluded, and all included views. With +previous:+, tables whose checksum is
|
|
49
|
+
# unchanged reuse the previous snapshot, so only changed tables are read again. Views and tables without a
|
|
50
|
+
# checksum are always read again.
|
|
48
51
|
#
|
|
49
52
|
# @param previous [Hash, nil] an earlier result of this method
|
|
50
53
|
# @return [Hash{String => Hash{String => TableSnap}}]
|
|
@@ -60,7 +63,8 @@ module TapeDeck
|
|
|
60
63
|
end
|
|
61
64
|
[table, snap]
|
|
62
65
|
end
|
|
63
|
-
[name,
|
|
66
|
+
views = view_names(db, name).to_h { [_1, dump(db, name, _1, nil, view: true)] }
|
|
67
|
+
[name, tables.merge(views)]
|
|
64
68
|
end
|
|
65
69
|
end
|
|
66
70
|
|
|
@@ -77,25 +81,78 @@ module TapeDeck
|
|
|
77
81
|
end
|
|
78
82
|
end
|
|
79
83
|
|
|
80
|
-
#
|
|
81
|
-
#
|
|
84
|
+
# Applies recording rules to the current schema of each database, by default the rules this recorder was created
|
|
85
|
+
# with. Other rules show what they would leave out and include.
|
|
82
86
|
#
|
|
83
87
|
# @param exclude_tables [Array<String>] table patterns, see {Configuration#exclude_tables}
|
|
84
88
|
# @param exclude_columns [Array<String>] column patterns, see {Configuration#exclude_columns}
|
|
85
|
-
# @
|
|
86
|
-
|
|
89
|
+
# @param include_views [Array<String>] view patterns, see {Configuration#include_views}
|
|
90
|
+
# @return [Hash{String => Hash{String => Array<String>}}] database -> +"excluded"+: sorted +"<table>"+ and
|
|
91
|
+
# +"<table>.<column>"+ names the exclude rules leave out, +"views"+: sorted names of the included views
|
|
92
|
+
def resolve(exclude_tables: @excluded_tables, exclude_columns: @excluded_columns, include_views: @included_views)
|
|
87
93
|
@dbs.to_h do |db_name, db|
|
|
88
94
|
tables = db.tables.map(&:to_s)
|
|
95
|
+
views = view_names(db, db_name, include_views)
|
|
89
96
|
table_names = tables.select { Patterns.match?(exclude_tables, "#{db_name}.#{_1}") }
|
|
90
|
-
column_names = (tables - table_names).flat_map do |table|
|
|
97
|
+
column_names = (tables - table_names + views).flat_map do |table|
|
|
91
98
|
excluded_column_names(db_name, table, db.schema(table.to_sym), exclude_columns).map { "#{table}.#{_1}" }
|
|
92
99
|
end
|
|
93
|
-
[db_name, (table_names + column_names).sort]
|
|
100
|
+
[db_name, {"excluded" => (table_names + column_names).sort, "views" => views}]
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Generated keys of the tables the diff inserts rows into: a single primary key column that is auto increment,
|
|
105
|
+
# or matches one of the patterns. Views have none.
|
|
106
|
+
#
|
|
107
|
+
# @param diff [Hash] result of {#diff}
|
|
108
|
+
# @param patterns [Array<String>] globs on +"<database>.<table>.<column>"+, see {Configuration#generated_keys}
|
|
109
|
+
# @return [Hash{String => Hash{String => String}}] database -> table -> key column
|
|
110
|
+
def generated_keys(diff, patterns: [])
|
|
111
|
+
diff.to_h do |db_name, tables|
|
|
112
|
+
db = @dbs.fetch(db_name)
|
|
113
|
+
real_tables = db.tables.map(&:to_s)
|
|
114
|
+
keys = tables.filter_map do |table, changes|
|
|
115
|
+
next unless changes.key?("inserted") && real_tables.include?(table)
|
|
116
|
+
|
|
117
|
+
pk = db.schema(table.to_sym).select { |_, info| info[:primary_key] }
|
|
118
|
+
next unless pk.size == 1
|
|
119
|
+
|
|
120
|
+
column, info = pk.first
|
|
121
|
+
generated = info[:auto_increment] || Patterns.match?(patterns, "#{db_name}.#{table}.#{column}")
|
|
122
|
+
[table, column.to_s] if generated
|
|
123
|
+
end
|
|
124
|
+
[db_name, keys.to_h]
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Single column references of the tables in the diff: declared foreign keys, and columns that match the patterns.
|
|
129
|
+
# Matching patterns win over declared foreign keys.
|
|
130
|
+
#
|
|
131
|
+
# @param diff [Hash] result of {#diff}
|
|
132
|
+
# @param patterns [Hash{String => Array<String>}] +"<database>.<table>"+ -> globs on +"<database>.<table>.<column>"+
|
|
133
|
+
# of columns holding its keys, see {Configuration#references_to}
|
|
134
|
+
# @return [Hash{String => Hash{String => Hash{String => Array(String, String)}}}] database -> table -> column ->
|
|
135
|
+
# +[database, table]+ it references
|
|
136
|
+
def references(diff, patterns: {})
|
|
137
|
+
diff.to_h do |db_name, tables|
|
|
138
|
+
db = @dbs.fetch(db_name)
|
|
139
|
+
real_tables = db.tables.map(&:to_s)
|
|
140
|
+
references = tables.keys.select { real_tables.include?(_1) }.to_h do |table|
|
|
141
|
+
declared = db.foreign_key_list(table.to_sym).filter_map do |foreign_key|
|
|
142
|
+
[foreign_key[:columns].first.to_s, [db_name, foreign_key[:table].to_s]] if foreign_key[:columns].size == 1
|
|
143
|
+
end
|
|
144
|
+
configured = db.schema(table.to_sym).filter_map do |column, _|
|
|
145
|
+
target = patterns.find { |_, columns| Patterns.match?(columns, "#{db_name}.#{table}.#{column}") }&.first
|
|
146
|
+
[column.to_s, target.split(".", 2)] if target
|
|
147
|
+
end
|
|
148
|
+
[table, declared.to_h.merge(configured.to_h)]
|
|
149
|
+
end
|
|
150
|
+
[db_name, references]
|
|
94
151
|
end
|
|
95
152
|
end
|
|
96
153
|
|
|
97
154
|
# @param db_name [String]
|
|
98
|
-
# @param name [String] +"<table>"+ or +"<table>.<column>"+, as returned by {#
|
|
155
|
+
# @param name [String] +"<table>"+ or +"<table>.<column>"+, as returned by {#resolve}
|
|
99
156
|
# @return [Boolean] whether the exclude rules leave this table or column out. Says nothing about primary keys,
|
|
100
157
|
# which are never left out.
|
|
101
158
|
def excluded?(db_name, name)
|
|
@@ -118,6 +175,13 @@ module TapeDeck
|
|
|
118
175
|
end
|
|
119
176
|
end
|
|
120
177
|
|
|
178
|
+
# @return [Array<String>] sorted names of the views the patterns include
|
|
179
|
+
def view_names(db, db_name, patterns = @included_views)
|
|
180
|
+
return [] if patterns.empty?
|
|
181
|
+
|
|
182
|
+
db.views.map(&:to_s).select { Patterns.match?(patterns, "#{db_name}.#{_1}") }.sort
|
|
183
|
+
end
|
|
184
|
+
|
|
121
185
|
# Diffs one database. A table missing on one side (created or dropped by the action) counts as empty there.
|
|
122
186
|
def tables_diff(before, after)
|
|
123
187
|
(before.keys | after.keys).sort.each_with_object({}) do |table, out|
|
|
@@ -129,23 +193,24 @@ module TapeDeck
|
|
|
129
193
|
end
|
|
130
194
|
end
|
|
131
195
|
|
|
132
|
-
# @return [Hash{String =>
|
|
196
|
+
# @return [Hash{String => Object, nil}] table name -> checksum, for the tables that are not excluded
|
|
133
197
|
def checksums(db, name)
|
|
134
198
|
tables = db.tables.map(&:to_s).sort.reject { excluded?(name, _1) }
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
list = tables.map { db.literal(Sequel.identifier(_1)) }.join(", ")
|
|
138
|
-
db.fetch("CHECKSUM TABLE #{list}").to_h do |r|
|
|
139
|
-
_db_name, table = r[:Table].split(".", 2) # MySQL reports "<db>.<table>"
|
|
140
|
-
[table, r[:Checksum]]
|
|
141
|
-
end
|
|
199
|
+
@adapters.fetch(name).checksums(tables)
|
|
142
200
|
end
|
|
143
201
|
|
|
202
|
+
# Views have no primary key. Whatever the schema says, their rows are compared as a multiset.
|
|
203
|
+
#
|
|
144
204
|
# @return [TableSnap]
|
|
145
|
-
def dump(db, name, table, sum)
|
|
205
|
+
def dump(db, name, table, sum, view: false)
|
|
206
|
+
adapter = @adapters.fetch(name)
|
|
146
207
|
schema = db.schema(table.to_sym)
|
|
147
|
-
pk =
|
|
148
|
-
|
|
208
|
+
pk = if view
|
|
209
|
+
[]
|
|
210
|
+
else
|
|
211
|
+
schema.select { |_, i| i[:primary_key] }.map(&:first)
|
|
212
|
+
end
|
|
213
|
+
columns = schema.to_h
|
|
149
214
|
|
|
150
215
|
drop = excluded_column_names(name, table, schema)
|
|
151
216
|
|
|
@@ -154,26 +219,16 @@ module TapeDeck
|
|
|
154
219
|
|
|
155
220
|
rows = {}
|
|
156
221
|
ds.each do |r|
|
|
157
|
-
row = r.except(*drop).to_h { |col, v| [col.to_s, serialize(v,
|
|
222
|
+
row = r.except(*drop).to_h { |col, v| [col.to_s, adapter.serialize(v, columns.fetch(col, {}))] }
|
|
158
223
|
if pk.empty?
|
|
159
224
|
rows[row] = rows.fetch(row, 0) + 1
|
|
160
225
|
else
|
|
161
|
-
rows[pk.map { serialize(r[_1],
|
|
226
|
+
rows[pk.map { adapter.serialize(r[_1], columns[_1]) }] = row
|
|
162
227
|
end
|
|
163
228
|
end
|
|
164
229
|
TableSnap.new(checksum: sum, pk: pk.map(&:to_s), rows: rows)
|
|
165
230
|
end
|
|
166
231
|
|
|
167
|
-
# @param info [Hash] column info from +db.schema+
|
|
168
|
-
# @return [Symbol, nil] Sequel's column type, +:time+ for all TIME columns, see {TIME_DB_TYPE}
|
|
169
|
-
def column_type(info)
|
|
170
|
-
if TIME_DB_TYPE.match?(info[:db_type].to_s)
|
|
171
|
-
:time
|
|
172
|
-
else
|
|
173
|
-
info[:type]
|
|
174
|
-
end
|
|
175
|
-
end
|
|
176
|
-
|
|
177
232
|
def empty_like(snap) = TableSnap.new(checksum: nil, pk: snap.pk, rows: {})
|
|
178
233
|
|
|
179
234
|
def table_diff(b, a)
|
|
@@ -216,55 +271,7 @@ module TapeDeck
|
|
|
216
271
|
compact("inserted" => sort_rows(inserted), "deleted" => sort_rows(deleted))
|
|
217
272
|
end
|
|
218
273
|
|
|
219
|
-
def sort_rows(rows) =
|
|
274
|
+
def sort_rows(rows) = Rows.sort(rows)
|
|
220
275
|
def compact(h) = h.reject { |_, v| v.empty? }
|
|
221
|
-
|
|
222
|
-
# Converts a column value to a YAML-stable scalar. +type+ is the column type from +db.schema+.
|
|
223
|
-
def serialize(v, type = nil)
|
|
224
|
-
case v
|
|
225
|
-
when String then maybe_utf8(v)
|
|
226
|
-
when BigDecimal then v.to_s("F")
|
|
227
|
-
when Sequel::SQLTime then time_of_day(v)
|
|
228
|
-
when Time then timestamp(v, type)
|
|
229
|
-
when Date then v.iso8601
|
|
230
|
-
else v # Integer, Float, true/false, nil
|
|
231
|
-
end
|
|
232
|
-
end
|
|
233
|
-
|
|
234
|
-
# mysql2 returns TIME columns as a +Time+ on 2000-01-01, not as +Sequel::SQLTime+. Those keep their local time of
|
|
235
|
-
# day, because converting them to UTC would shift the hour.
|
|
236
|
-
def timestamp(v, type)
|
|
237
|
-
if type == :time
|
|
238
|
-
time_of_day(v)
|
|
239
|
-
else
|
|
240
|
-
v.utc.iso8601(6)
|
|
241
|
-
end
|
|
242
|
-
end
|
|
243
|
-
|
|
244
|
-
def time_of_day(v) = v.strftime("%H:%M:%S.%6N")
|
|
245
|
-
|
|
246
|
-
# Small binaries encode as hex. Large ones are shown as a digest so the YAML stays small.
|
|
247
|
-
def bin(v)
|
|
248
|
-
if v.bytesize <= 64
|
|
249
|
-
"0x#{v.unpack1("H*")}"
|
|
250
|
-
else
|
|
251
|
-
"sha256:#{Digest::SHA256.hexdigest(v)}"
|
|
252
|
-
end
|
|
253
|
-
end
|
|
254
|
-
|
|
255
|
-
# BINARY, VARBINARY and BLOB columns arrive as ASCII-8BIT strings (or Sequel::SQL::Blob, depending on the adapter).
|
|
256
|
-
# Ones that hold valid UTF-8 become plain UTF-8 strings, because YAML would otherwise store them as base64
|
|
257
|
-
# (+!binary+). Everything else goes through {#bin}.
|
|
258
|
-
def maybe_utf8(v)
|
|
259
|
-
binary = v.encoding == Encoding::BINARY || v.is_a?(Sequel::SQL::Blob)
|
|
260
|
-
return v unless binary
|
|
261
|
-
|
|
262
|
-
utf8 = String.new(v, encoding: Encoding::UTF_8)
|
|
263
|
-
if utf8.valid_encoding?
|
|
264
|
-
utf8
|
|
265
|
-
else
|
|
266
|
-
bin(v)
|
|
267
|
-
end
|
|
268
|
-
end
|
|
269
276
|
end
|
|
270
277
|
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module TapeDeck
|
|
4
|
+
# A stable order for rows, so tapes come out the same on every Ruby version. Rows hold plain values only, see
|
|
5
|
+
# {Adapters::Base#serialize}, and the JSON format pins down how those are written. +inspect+ does not, it changed
|
|
6
|
+
# in Ruby 3.4.
|
|
7
|
+
#
|
|
8
|
+
# @api private
|
|
9
|
+
module Rows
|
|
10
|
+
# @param rows [Array<Hash>]
|
|
11
|
+
# @return [Array<Hash>] sorted by their JSON
|
|
12
|
+
def self.sort(rows)
|
|
13
|
+
rows.sort_by { JSON.generate(_1, allow_nan: true) }
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|