tape_deck 0.1.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.devcontainer/compose.yml +33 -0
- data/.devcontainer/devcontainer.json +13 -0
- data/.yardopts +7 -0
- data/CHANGELOG.md +5 -0
- data/CODE_OF_CONDUCT.md +10 -0
- data/DEVELOPMENT.md +86 -0
- data/LICENSE.txt +21 -0
- data/README.md +158 -0
- data/Rakefile +85 -0
- data/lib/tape_deck/configuration.rb +113 -0
- data/lib/tape_deck/errors.rb +54 -0
- data/lib/tape_deck/filter.rb +72 -0
- data/lib/tape_deck/patterns.rb +21 -0
- data/lib/tape_deck/recorder.rb +270 -0
- data/lib/tape_deck/session.rb +100 -0
- data/lib/tape_deck/tape.rb +152 -0
- data/lib/tape_deck/version.rb +6 -0
- data/lib/tape_deck.rb +295 -0
- data/sig/tape_deck.rbs +4 -0
- metadata +95 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module TapeDeck
|
|
4
|
+
# Narrows a diff to what one use of a tape wants compared, see "Excluding and ignoring" in {TapeDeck}.
|
|
5
|
+
#
|
|
6
|
+
# @api private
|
|
7
|
+
class Filter
|
|
8
|
+
# @param only [Array<String>, String, nil] patterns to keep, everything if nil
|
|
9
|
+
# @param ignore [Array<String>, String, nil] patterns to drop
|
|
10
|
+
def initialize(only: nil, ignore: nil)
|
|
11
|
+
@only = Array(only).map(&:to_s)
|
|
12
|
+
@ignore = Array(ignore).map(&:to_s)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# @param patterns [Array<String>] more patterns to drop
|
|
16
|
+
# @return [Filter] a copy that also drops these
|
|
17
|
+
def ignoring(patterns)
|
|
18
|
+
Filter.new(only: @only, ignore: @ignore + patterns)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# @param diff [Hash] a diff as stored on a tape
|
|
22
|
+
# @return [Hash] the diff without ignored columns, and without rows, tables and databases left empty by that
|
|
23
|
+
def apply(diff)
|
|
24
|
+
return diff if @only.empty? && @ignore.empty?
|
|
25
|
+
|
|
26
|
+
diff.each_with_object({}) do |(db_name, tables), out|
|
|
27
|
+
kept_tables = tables.each_with_object({}) do |(table, changes), kept|
|
|
28
|
+
kept_changes = filter_table("#{db_name}.#{table}", changes)
|
|
29
|
+
kept[table] = kept_changes unless kept_changes.empty?
|
|
30
|
+
end
|
|
31
|
+
out[db_name] = kept_tables unless kept_tables.empty?
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def filter_table(table_name, changes)
|
|
38
|
+
kept = {
|
|
39
|
+
"inserted" => filter_rows(table_name, changes.fetch("inserted", [])),
|
|
40
|
+
"updated" => filter_updates(table_name, changes.fetch("updated", [])),
|
|
41
|
+
"deleted" => filter_rows(table_name, changes.fetch("deleted", []))
|
|
42
|
+
}
|
|
43
|
+
kept.reject { |_, entries| entries.empty? }
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Rows are sorted again, because their recorded order may depend on values that are now ignored.
|
|
47
|
+
def filter_rows(table_name, rows)
|
|
48
|
+
kept = rows.map { |row| filter_columns(table_name, row) }
|
|
49
|
+
kept.reject(&:empty?).sort_by(&:inspect)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# The "key" of an update is kept as it is, because it identifies the row.
|
|
53
|
+
def filter_updates(table_name, updates)
|
|
54
|
+
updates.filter_map do |update|
|
|
55
|
+
changes = filter_columns(table_name, update.fetch("changes"))
|
|
56
|
+
update.merge("changes" => changes) unless changes.empty?
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def filter_columns(table_name, columns)
|
|
61
|
+
columns.select { |column, _| compare?(table_name, "#{table_name}.#{column}") }
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# A column is addressed by a pattern for the column itself or one for its whole table.
|
|
65
|
+
def compare?(table_name, column_name)
|
|
66
|
+
names = [table_name, column_name]
|
|
67
|
+
return false if names.any? { Patterns.match?(@ignore, _1) }
|
|
68
|
+
|
|
69
|
+
@only.empty? || names.any? { Patterns.match?(@only, _1) }
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module TapeDeck
|
|
4
|
+
# Glob matching for table and column names, see "Patterns" in {TapeDeck}.
|
|
5
|
+
#
|
|
6
|
+
# @api private
|
|
7
|
+
module Patterns
|
|
8
|
+
# @param patterns [Array<String>]
|
|
9
|
+
# @param name [String] +"<database>.<table>"+ or +"<database>.<table>.<column>"+
|
|
10
|
+
# @return [Boolean] whether any of the patterns matches the name
|
|
11
|
+
def self.match?(patterns, name)
|
|
12
|
+
patterns.any? { File.fnmatch?(_1, name, File::FNM_EXTGLOB) }
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# @param name [String]
|
|
16
|
+
# @return [String] a pattern that matches only this name
|
|
17
|
+
def self.escape(name)
|
|
18
|
+
name.gsub(/[*?\[\]{}\\]/) { "\\#{_1}" }
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module TapeDeck
|
|
4
|
+
# Snapshot of one table.
|
|
5
|
+
#
|
|
6
|
+
# @!attribute [r] checksum
|
|
7
|
+
# @return [Integer, nil] result of +CHECKSUM TABLE+
|
|
8
|
+
# @!attribute [r] pk
|
|
9
|
+
# @return [Array<String>] primary key columns, empty if the table has none
|
|
10
|
+
# @!attribute [r] rows
|
|
11
|
+
# @return [Hash] +{ [pk values] => row }+ for tables with a primary key, +{ row => occurrence count }+ for tables
|
|
12
|
+
# without one
|
|
13
|
+
# @api private
|
|
14
|
+
TableSnap = Data.define(:checksum, :pk, :rows)
|
|
15
|
+
|
|
16
|
+
# Takes snapshots of all configured databases and diffs them. Used by {Session}, but usable on its own when you don't
|
|
17
|
+
# need tape files.
|
|
18
|
+
class Recorder
|
|
19
|
+
# Database types of TIME columns, with or without fractional seconds. Sequel only types +time+ as +:time+, not
|
|
20
|
+
# +time(6)+.
|
|
21
|
+
TIME_DB_TYPE = /\Atime\b/i
|
|
22
|
+
|
|
23
|
+
# @return [Array<String>] see {Configuration#exclude_tables}
|
|
24
|
+
attr_reader :excluded_tables
|
|
25
|
+
|
|
26
|
+
# @return [Array<String>] see {Configuration#exclude_columns}
|
|
27
|
+
attr_reader :excluded_columns
|
|
28
|
+
|
|
29
|
+
# @param dbs [Hash{String => Sequel::Database}]
|
|
30
|
+
# @param exclude_tables [Array<String>] see {Configuration#exclude_tables}
|
|
31
|
+
# @param exclude_columns [Array<String>] see {Configuration#exclude_columns}
|
|
32
|
+
def initialize(dbs, exclude_tables: [], exclude_columns: [])
|
|
33
|
+
@dbs = dbs
|
|
34
|
+
@excluded_tables = exclude_tables
|
|
35
|
+
@excluded_columns = exclude_columns
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# @yield the action whose database changes are recorded
|
|
39
|
+
# @return [Hash] the diff, see {#diff}
|
|
40
|
+
def record
|
|
41
|
+
before = snapshot
|
|
42
|
+
yield
|
|
43
|
+
diff(before, snapshot(previous: before))
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Reads all tables that are not excluded. With +previous:+, tables whose +CHECKSUM TABLE+ is unchanged reuse the
|
|
47
|
+
# previous snapshot, so only changed tables are read again.
|
|
48
|
+
#
|
|
49
|
+
# @param previous [Hash, nil] an earlier result of this method
|
|
50
|
+
# @return [Hash{String => Hash{String => TableSnap}}]
|
|
51
|
+
def snapshot(previous: nil)
|
|
52
|
+
@dbs.to_h do |name, db|
|
|
53
|
+
tables = checksums(db, name).to_h do |table, sum|
|
|
54
|
+
prev = previous&.dig(name, table)
|
|
55
|
+
unchanged = prev && sum && prev.checksum == sum
|
|
56
|
+
snap = if unchanged
|
|
57
|
+
prev
|
|
58
|
+
else
|
|
59
|
+
dump(db, name, table, sum)
|
|
60
|
+
end
|
|
61
|
+
[table, snap]
|
|
62
|
+
end
|
|
63
|
+
[name, tables]
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Rows are matched by primary key. Tables without one are compared as a multiset of rows (+inserted+/+deleted+
|
|
68
|
+
# only). Unchanged tables and empty sections are omitted.
|
|
69
|
+
#
|
|
70
|
+
# @param before [Hash] result of {#snapshot}
|
|
71
|
+
# @param after [Hash] result of {#snapshot}
|
|
72
|
+
# @return [Hash{String => Hash{String => Hash}}] database -> table -> +"inserted"+ / +"updated"+ / +"deleted"+
|
|
73
|
+
def diff(before, after)
|
|
74
|
+
before.keys.sort.each_with_object({}) do |db_name, out|
|
|
75
|
+
changed_tables = tables_diff(before.fetch(db_name), after.fetch(db_name))
|
|
76
|
+
out[db_name] = changed_tables unless changed_tables.empty?
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# What exclude rules leave out of each database, by default the ones this recorder was created with. Reads the
|
|
81
|
+
# current schema, so other rules show what they would have left out.
|
|
82
|
+
#
|
|
83
|
+
# @param exclude_tables [Array<String>] table patterns, see {Configuration#exclude_tables}
|
|
84
|
+
# @param exclude_columns [Array<String>] column patterns, see {Configuration#exclude_columns}
|
|
85
|
+
# @return [Hash{String => Array<String>}] database -> sorted +"<table>"+ and +"<table>.<column>"+ names
|
|
86
|
+
def exclusions(exclude_tables: @excluded_tables, exclude_columns: @excluded_columns)
|
|
87
|
+
@dbs.to_h do |db_name, db|
|
|
88
|
+
tables = db.tables.map(&:to_s)
|
|
89
|
+
table_names = tables.select { Patterns.match?(exclude_tables, "#{db_name}.#{_1}") }
|
|
90
|
+
column_names = (tables - table_names).flat_map do |table|
|
|
91
|
+
excluded_column_names(db_name, table, db.schema(table.to_sym), exclude_columns).map { "#{table}.#{_1}" }
|
|
92
|
+
end
|
|
93
|
+
[db_name, (table_names + column_names).sort]
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# @param db_name [String]
|
|
98
|
+
# @param name [String] +"<table>"+ or +"<table>.<column>"+, as returned by {#exclusions}
|
|
99
|
+
# @return [Boolean] whether the exclude rules leave this table or column out. Says nothing about primary keys,
|
|
100
|
+
# which are never left out.
|
|
101
|
+
def excluded?(db_name, name)
|
|
102
|
+
patterns = if name.include?(".")
|
|
103
|
+
@excluded_columns
|
|
104
|
+
else
|
|
105
|
+
@excluded_tables
|
|
106
|
+
end
|
|
107
|
+
Patterns.match?(patterns, "#{db_name}.#{name}")
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
private
|
|
111
|
+
|
|
112
|
+
# Primary key columns are never excluded, because rows are matched by them.
|
|
113
|
+
#
|
|
114
|
+
# @return [Array<Symbol>]
|
|
115
|
+
def excluded_column_names(db_name, table, schema, patterns = @excluded_columns)
|
|
116
|
+
schema.filter_map do |col, info|
|
|
117
|
+
col if !info[:primary_key] && Patterns.match?(patterns, "#{db_name}.#{table}.#{col}")
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Diffs one database. A table missing on one side (created or dropped by the action) counts as empty there.
|
|
122
|
+
def tables_diff(before, after)
|
|
123
|
+
(before.keys | after.keys).sort.each_with_object({}) do |table, out|
|
|
124
|
+
b, a = before[table], after[table]
|
|
125
|
+
next if b.equal?(a) # checksum unchanged, snapshot reused
|
|
126
|
+
|
|
127
|
+
changes = table_diff(b || empty_like(a), a || empty_like(b))
|
|
128
|
+
out[table] = changes unless changes.empty?
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# @return [Hash{String => Integer, nil}] table name -> checksum
|
|
133
|
+
def checksums(db, name)
|
|
134
|
+
tables = db.tables.map(&:to_s).sort.reject { excluded?(name, _1) }
|
|
135
|
+
return {} if tables.empty?
|
|
136
|
+
|
|
137
|
+
list = tables.map { db.literal(Sequel.identifier(_1)) }.join(", ")
|
|
138
|
+
db.fetch("CHECKSUM TABLE #{list}").to_h do |r|
|
|
139
|
+
_db_name, table = r[:Table].split(".", 2) # MySQL reports "<db>.<table>"
|
|
140
|
+
[table, r[:Checksum]]
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# @return [TableSnap]
|
|
145
|
+
def dump(db, name, table, sum)
|
|
146
|
+
schema = db.schema(table.to_sym)
|
|
147
|
+
pk = schema.select { |_, i| i[:primary_key] }.map(&:first)
|
|
148
|
+
types = schema.to_h { |col, i| [col, column_type(i)] }
|
|
149
|
+
|
|
150
|
+
drop = excluded_column_names(name, table, schema)
|
|
151
|
+
|
|
152
|
+
ds = db[table.to_sym]
|
|
153
|
+
ds = ds.order(*pk) unless pk.empty?
|
|
154
|
+
|
|
155
|
+
rows = {}
|
|
156
|
+
ds.each do |r|
|
|
157
|
+
row = r.except(*drop).to_h { |col, v| [col.to_s, serialize(v, types[col])] }
|
|
158
|
+
if pk.empty?
|
|
159
|
+
rows[row] = rows.fetch(row, 0) + 1
|
|
160
|
+
else
|
|
161
|
+
rows[pk.map { serialize(r[_1], types[_1]) }] = row
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
TableSnap.new(checksum: sum, pk: pk.map(&:to_s), rows: rows)
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# @param info [Hash] column info from +db.schema+
|
|
168
|
+
# @return [Symbol, nil] Sequel's column type, +:time+ for all TIME columns, see {TIME_DB_TYPE}
|
|
169
|
+
def column_type(info)
|
|
170
|
+
if TIME_DB_TYPE.match?(info[:db_type].to_s)
|
|
171
|
+
:time
|
|
172
|
+
else
|
|
173
|
+
info[:type]
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def empty_like(snap) = TableSnap.new(checksum: nil, pk: snap.pk, rows: {})
|
|
178
|
+
|
|
179
|
+
def table_diff(b, a)
|
|
180
|
+
if b.pk.empty?
|
|
181
|
+
multiset_diff(b.rows, a.rows)
|
|
182
|
+
else
|
|
183
|
+
keyed_diff(b, a)
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
def keyed_diff(b, a)
|
|
188
|
+
inserted = (a.rows.keys - b.rows.keys).map { a.rows[_1] }
|
|
189
|
+
deleted = (b.rows.keys - a.rows.keys).map { b.rows[_1] }
|
|
190
|
+
updated = (a.rows.keys & b.rows.keys).filter_map do |k|
|
|
191
|
+
old, new = b.rows[k], a.rows[k]
|
|
192
|
+
next if old == new
|
|
193
|
+
|
|
194
|
+
{"key" => a.pk.zip(k).to_h, "changes" => changed_columns(old, new)}
|
|
195
|
+
end
|
|
196
|
+
compact("inserted" => inserted, "updated" => updated, "deleted" => deleted)
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# @return [Hash{String => Array(Object, Object)}] column -> +[old, new]+ for columns whose value differs
|
|
200
|
+
def changed_columns(old, new)
|
|
201
|
+
(old.keys | new.keys).each_with_object({}) do |col, changes|
|
|
202
|
+
changes[col] = [old[col], new[col]] unless old[col] == new[col]
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def multiset_diff(b, a)
|
|
207
|
+
inserted, deleted = [], []
|
|
208
|
+
(b.keys | a.keys).each do |row|
|
|
209
|
+
delta = a.fetch(row, 0) - b.fetch(row, 0)
|
|
210
|
+
if delta.positive?
|
|
211
|
+
inserted.concat([row] * delta)
|
|
212
|
+
elsif delta.negative?
|
|
213
|
+
deleted.concat([row] * delta.abs)
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
compact("inserted" => sort_rows(inserted), "deleted" => sort_rows(deleted))
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def sort_rows(rows) = rows.sort_by(&:inspect)
|
|
220
|
+
def compact(h) = h.reject { |_, v| v.empty? }
|
|
221
|
+
|
|
222
|
+
# Converts a column value to a YAML-stable scalar. +type+ is the column type from +db.schema+.
|
|
223
|
+
def serialize(v, type = nil)
|
|
224
|
+
case v
|
|
225
|
+
when String then maybe_utf8(v)
|
|
226
|
+
when BigDecimal then v.to_s("F")
|
|
227
|
+
when Sequel::SQLTime then time_of_day(v)
|
|
228
|
+
when Time then timestamp(v, type)
|
|
229
|
+
when Date then v.iso8601
|
|
230
|
+
else v # Integer, Float, true/false, nil
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# mysql2 returns TIME columns as a +Time+ on 2000-01-01, not as +Sequel::SQLTime+. Those keep their local time of
|
|
235
|
+
# day, because converting them to UTC would shift the hour.
|
|
236
|
+
def timestamp(v, type)
|
|
237
|
+
if type == :time
|
|
238
|
+
time_of_day(v)
|
|
239
|
+
else
|
|
240
|
+
v.utc.iso8601(6)
|
|
241
|
+
end
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
def time_of_day(v) = v.strftime("%H:%M:%S.%6N")
|
|
245
|
+
|
|
246
|
+
# Small binaries encode as hex. Large ones are shown as a digest so the YAML stays small.
|
|
247
|
+
def bin(v)
|
|
248
|
+
if v.bytesize <= 64
|
|
249
|
+
"0x#{v.unpack1("H*")}"
|
|
250
|
+
else
|
|
251
|
+
"sha256:#{Digest::SHA256.hexdigest(v)}"
|
|
252
|
+
end
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
# BINARY, VARBINARY and BLOB columns arrive as ASCII-8BIT strings (or Sequel::SQL::Blob, depending on the adapter).
|
|
256
|
+
# Ones that hold valid UTF-8 become plain UTF-8 strings, because YAML would otherwise store them as base64
|
|
257
|
+
# (+!binary+). Everything else goes through {#bin}.
|
|
258
|
+
def maybe_utf8(v)
|
|
259
|
+
binary = v.encoding == Encoding::BINARY || v.is_a?(Sequel::SQL::Blob)
|
|
260
|
+
return v unless binary
|
|
261
|
+
|
|
262
|
+
utf8 = String.new(v, encoding: Encoding::UTF_8)
|
|
263
|
+
if utf8.valid_encoding?
|
|
264
|
+
utf8
|
|
265
|
+
else
|
|
266
|
+
bin(v)
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
end
|
|
270
|
+
end
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module TapeDeck
|
|
4
|
+
# One run of one tape: takes the "before" snapshot when created, and on {#finish} captures the diff and records it
|
|
5
|
+
# to the tape or verifies it against the tape.
|
|
6
|
+
#
|
|
7
|
+
# @api private
|
|
8
|
+
class Session
|
|
9
|
+
# @return [Tape]
|
|
10
|
+
attr_reader :tape
|
|
11
|
+
|
|
12
|
+
# @param tape [Tape]
|
|
13
|
+
# @param recorder [Recorder]
|
|
14
|
+
# @param filter [Filter] narrows what is compared when verifying
|
|
15
|
+
# @param on_schema_change [Symbol] see {Configuration#on_schema_change}
|
|
16
|
+
def initialize(tape, recorder, filter, on_schema_change: :error)
|
|
17
|
+
@tape = tape
|
|
18
|
+
@recorder = recorder
|
|
19
|
+
@filter = filter
|
|
20
|
+
@on_schema_change = on_schema_change
|
|
21
|
+
@before = recorder.snapshot
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# @param discard [Boolean] neither record nor verify, e.g. because the block failed
|
|
25
|
+
# @return [void]
|
|
26
|
+
# @raise [UnexpectedDiffError] if verifying and the diff does not match the tape
|
|
27
|
+
# @raise [IncompatibleTapeError] if verifying and the tape lacks data that is no longer excluded, or the schema
|
|
28
|
+
# changed and {Configuration#on_schema_change} is +:error+
|
|
29
|
+
# @raise [TapeLoadingError] if verifying and the tape cannot be read, see {Tape#read}
|
|
30
|
+
def finish(discard: false)
|
|
31
|
+
return if discard
|
|
32
|
+
|
|
33
|
+
after = @recorder.snapshot(previous: @before)
|
|
34
|
+
actual = round_trip(@recorder.diff(@before, after))
|
|
35
|
+
exclusions = @recorder.exclusions
|
|
36
|
+
if tape.recording?
|
|
37
|
+
tape.write(actual, recorded: {
|
|
38
|
+
"exclude_tables" => @recorder.excluded_tables,
|
|
39
|
+
"exclude_columns" => @recorder.excluded_columns,
|
|
40
|
+
"databases" => exclusions
|
|
41
|
+
})
|
|
42
|
+
else
|
|
43
|
+
verify(actual, exclusions)
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
# @return [Hash] the diff with the same shape as a tape read from disk
|
|
50
|
+
def round_trip(diff) = YAML.safe_load(Tape.serialize(diff))
|
|
51
|
+
|
|
52
|
+
# Compares the watched databases only. What is excluded now, but is on the tape, is dropped from both sides.
|
|
53
|
+
def verify(actual, exclusions)
|
|
54
|
+
recorded = tape.read.fetch("recorded")
|
|
55
|
+
tape_exclusions = recorded.fetch("databases").slice(*exclusions.keys)
|
|
56
|
+
report_schema_changes(recorded, tape_exclusions)
|
|
57
|
+
filter = @filter.ignoring(newly_excluded(tape_exclusions, exclusions))
|
|
58
|
+
expected = filter.apply(tape.read.fetch("diff").slice(*exclusions.keys))
|
|
59
|
+
actual = filter.apply(actual)
|
|
60
|
+
raise UnexpectedDiffError.new(tape, expected, actual) unless expected == actual
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# The recorded exclude rules, applied to the current schema, must leave out what they did when recording.
|
|
64
|
+
def report_schema_changes(recorded, tape_exclusions)
|
|
65
|
+
return if @on_schema_change == :ignore
|
|
66
|
+
|
|
67
|
+
now = @recorder.exclusions(
|
|
68
|
+
exclude_tables: recorded.fetch("exclude_tables"),
|
|
69
|
+
exclude_columns: recorded.fetch("exclude_columns")
|
|
70
|
+
)
|
|
71
|
+
changes = tape_exclusions.filter_map do |db_name, names|
|
|
72
|
+
added = now.fetch(db_name) - names
|
|
73
|
+
removed = names - now.fetch(db_name)
|
|
74
|
+
"#{db_name}: added #{added.inspect}, removed #{removed.inspect}" unless added.empty? && removed.empty?
|
|
75
|
+
end
|
|
76
|
+
return if changes.empty?
|
|
77
|
+
|
|
78
|
+
message = "schema changed since tape #{tape.file} was recorded, its exclude rules now match " \
|
|
79
|
+
"other tables or columns (#{changes.join(", ")})"
|
|
80
|
+
raise IncompatibleTapeError, message if @on_schema_change == :error
|
|
81
|
+
|
|
82
|
+
warn "TapeDeck: #{message}"
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# @return [Array<String>] patterns for what is excluded now, but was recorded on the tape
|
|
86
|
+
# @raise [IncompatibleTapeError] if the tape left out something that is no longer excluded
|
|
87
|
+
def newly_excluded(tape_exclusions, exclusions)
|
|
88
|
+
exclusions.flat_map do |db_name, names|
|
|
89
|
+
tape_names = tape_exclusions.fetch(db_name)
|
|
90
|
+
no_longer_excluded = tape_names.reject { @recorder.excluded?(db_name, _1) }
|
|
91
|
+
unless no_longer_excluded.empty?
|
|
92
|
+
raise IncompatibleTapeError, "tape #{tape.file} left out #{no_longer_excluded.inspect} of #{db_name}, " \
|
|
93
|
+
"which is no longer excluded. Re-record the tape."
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
(names - tape_names).map { Patterns.escape("#{db_name}.#{_1}") }
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module TapeDeck
|
|
4
|
+
# One recorded diff, stored as a YAML file. Like a +File+, a tape knows where it lives, whether it is there to be
|
|
5
|
+
# recorded or verified, and how to read and write itself. It never touches a database.
|
|
6
|
+
#
|
|
7
|
+
# Next to the diff, a tape stores what was recorded: the exclude rules, the watched databases, and per database the
|
|
8
|
+
# tables and columns the rules left out. Empty lists are left out of the file.
|
|
9
|
+
#
|
|
10
|
+
# recorded:
|
|
11
|
+
# exclude_tables: [shop.sessions]
|
|
12
|
+
# exclude_columns: ["*.created_at"]
|
|
13
|
+
# databases:
|
|
14
|
+
# shop:
|
|
15
|
+
# excluded: [orders.created_at, sessions]
|
|
16
|
+
# billing:
|
|
17
|
+
# diff:
|
|
18
|
+
# shop:
|
|
19
|
+
# orders:
|
|
20
|
+
# inserted: [...]
|
|
21
|
+
class Tape
|
|
22
|
+
# Lists in the +recorded+ section of a tape file, written on one line.
|
|
23
|
+
RECORDED_LISTS = %w[exclude_tables exclude_columns excluded].freeze
|
|
24
|
+
|
|
25
|
+
# Supported values for the +:record+ option:
|
|
26
|
+
# [+:once+] record if the tape file is missing, otherwise verify
|
|
27
|
+
# [+:verify+] verify only, never record. A missing tape raises {TapeLoadingError}
|
|
28
|
+
# [+:overwrite+] always record, replacing an existing tape
|
|
29
|
+
RECORD_MODES = %i[once verify overwrite].freeze
|
|
30
|
+
|
|
31
|
+
# Characters in a tape name that are replaced with +_+ in its file name.
|
|
32
|
+
UNSAFE_FILE_NAME_CHARS = %r{[^\w\-/]+}
|
|
33
|
+
|
|
34
|
+
# @return [String]
|
|
35
|
+
attr_reader :name
|
|
36
|
+
|
|
37
|
+
# @return [Symbol] one of {RECORD_MODES}
|
|
38
|
+
attr_reader :record_mode
|
|
39
|
+
|
|
40
|
+
# @param diff [Hash] a diff as returned by {Recorder#diff}
|
|
41
|
+
# @return [String] the YAML stored in a tape file
|
|
42
|
+
def self.serialize(diff) = YAML.dump(diff, line_width: -1)
|
|
43
|
+
|
|
44
|
+
# @param mode [Symbol]
|
|
45
|
+
# @return [void]
|
|
46
|
+
# @raise [ArgumentError] unless the mode is one of {RECORD_MODES}
|
|
47
|
+
def self.validate_record_mode(mode)
|
|
48
|
+
return if RECORD_MODES.include?(mode)
|
|
49
|
+
|
|
50
|
+
raise ArgumentError, "invalid record mode #{mode.inspect}, use one of #{RECORD_MODES.inspect}"
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Like {.serialize}, but writes the {RECORDED_LISTS} on one line, so the +recorded+ section stays short.
|
|
54
|
+
#
|
|
55
|
+
# @param content [Hash] see {#read}
|
|
56
|
+
# @return [String] the YAML stored in a tape file
|
|
57
|
+
def self.dump(content)
|
|
58
|
+
tree = Psych::Visitors::YAMLTree.create(line_width: -1)
|
|
59
|
+
tree << content
|
|
60
|
+
tree.tree.each do |node|
|
|
61
|
+
next unless node.is_a?(Psych::Nodes::Mapping)
|
|
62
|
+
|
|
63
|
+
node.children.each_slice(2) do |key, value|
|
|
64
|
+
recorded_list = RECORDED_LISTS.include?(key.value) && value.is_a?(Psych::Nodes::Sequence)
|
|
65
|
+
value.style = Psych::Nodes::Sequence::FLOW if recorded_list
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
tree.tree.yaml(nil, line_width: -1)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# @param name [String] tape name, may contain +/+ to create subdirectories
|
|
72
|
+
# @param record [Symbol] one of {RECORD_MODES}, defaults to {Configuration#default_record_mode}
|
|
73
|
+
# @raise [ArgumentError] for an unknown record mode
|
|
74
|
+
# @raise [TapeLoadingError] if the tape must be verified but does not exist
|
|
75
|
+
def initialize(name, record: TapeDeck.configuration.default_record_mode)
|
|
76
|
+
self.class.validate_record_mode(record)
|
|
77
|
+
@name = name.to_s
|
|
78
|
+
@record_mode = record
|
|
79
|
+
|
|
80
|
+
tape_exists = File.exist?(file)
|
|
81
|
+
raise TapeLoadingError, "no tape at #{file} (record mode :verify)" if @record_mode == :verify && !tape_exists
|
|
82
|
+
|
|
83
|
+
@recording = @record_mode == :overwrite || !tape_exists
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# @return [Boolean] whether the tape is to be written
|
|
87
|
+
def recording? = @recording
|
|
88
|
+
|
|
89
|
+
# @return [Boolean] whether the tape is to be read and compared
|
|
90
|
+
def verifying? = !recording?
|
|
91
|
+
|
|
92
|
+
# @return [String] path of the tape file, see {#file_name}
|
|
93
|
+
def file
|
|
94
|
+
File.join(TapeDeck.configuration.tape_library_dir, file_name)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# @return [String] the tape name with {UNSAFE_FILE_NAME_CHARS} replaced, plus +.yml+
|
|
98
|
+
def file_name
|
|
99
|
+
"#{name.gsub(UNSAFE_FILE_NAME_CHARS, "_")}.yml"
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Reads the tape file once, later calls return the same content.
|
|
103
|
+
#
|
|
104
|
+
# @return [Hash] +"recorded"+: see {#write}, and +"diff"+: the stored diff
|
|
105
|
+
# @raise [TapeLoadingError] if the file is not valid YAML or does not hold a recording
|
|
106
|
+
def read
|
|
107
|
+
@content ||= load
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# @param diff [Hash] the diff to store in the tape file, replacing its content
|
|
111
|
+
# @param recorded [Hash] +"exclude_tables"+ and +"exclude_columns"+: the exclude rules, +"databases"+: database ->
|
|
112
|
+
# names the rules left out of it, as returned by {Recorder#exclusions}
|
|
113
|
+
# @return [void]
|
|
114
|
+
def write(diff, recorded:)
|
|
115
|
+
databases = recorded.fetch("databases").transform_values do |names|
|
|
116
|
+
{"excluded" => names} unless names.empty?
|
|
117
|
+
end
|
|
118
|
+
patterns = recorded.slice("exclude_tables", "exclude_columns").reject { |_, list| list.empty? }
|
|
119
|
+
FileUtils.mkdir_p(File.dirname(file))
|
|
120
|
+
File.write(file, self.class.dump("recorded" => {**patterns, "databases" => databases}, "diff" => diff))
|
|
121
|
+
@content = nil
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
private
|
|
125
|
+
|
|
126
|
+
def load
|
|
127
|
+
content = YAML.safe_load_file(file)
|
|
128
|
+
raise TapeLoadingError, "tape #{file} does not hold a recording" unless valid_content?(content)
|
|
129
|
+
|
|
130
|
+
recorded = content["recorded"]
|
|
131
|
+
databases = recorded["databases"].transform_values { Array(_1&.dig("excluded")) }
|
|
132
|
+
{
|
|
133
|
+
"recorded" => {
|
|
134
|
+
"exclude_tables" => Array(recorded["exclude_tables"]),
|
|
135
|
+
"exclude_columns" => Array(recorded["exclude_columns"]),
|
|
136
|
+
"databases" => databases
|
|
137
|
+
},
|
|
138
|
+
"diff" => content["diff"]
|
|
139
|
+
}
|
|
140
|
+
rescue Psych::Exception => e
|
|
141
|
+
raise TapeLoadingError, "cannot load tape #{file}: #{e.message}"
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def valid_content?(content)
|
|
145
|
+
return false unless content.is_a?(Hash) && content["diff"].is_a?(Hash)
|
|
146
|
+
|
|
147
|
+
recorded = content["recorded"]
|
|
148
|
+
databases = recorded["databases"] if recorded.is_a?(Hash)
|
|
149
|
+
databases.is_a?(Hash) && databases.values.all? { _1.nil? || _1.is_a?(Hash) }
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
end
|