tape_deck 0.1.0.beta1 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "adapters/base"
4
+ require_relative "adapters/mysql"
5
+ require_relative "adapters/postgres"
6
+ require_relative "adapters/sqlite"
7
+
8
+ module TapeDeck
9
+ # The database specific parts of taking snapshots: telling which tables changed, and turning column values into
10
+ # plain YAML values. Everything else goes through Sequel and works the same for every database.
11
+ module Adapters
12
+ # Adapters by +Sequel::Database#database_type+.
13
+ BY_DATABASE_TYPE = {mysql: MySQL, postgres: Postgres, sqlite: SQLite}.freeze
14
+
15
+ # @param db [Sequel::Database]
16
+ # @return [Base] the adapter for the database type. {Base} for other databases, which works, but reads every
17
+ # table on every snapshot.
18
+ def self.for(db)
19
+ BY_DATABASE_TYPE.fetch(db.database_type, Base).new(db)
20
+ end
21
+ end
22
+ end
@@ -33,6 +33,21 @@ module TapeDeck
33
33
  # @return [Array<String>] patterns added with {#exclude_columns}
34
34
  attr_reader :excluded_columns
35
35
 
36
+ # @return [Array<String>] patterns added with {#include_views}
37
+ attr_reader :included_views
38
+
39
+ # @return [Array<String>] patterns added with {#keep_new_ids}
40
+ attr_reader :kept_new_ids
41
+
42
+ # @return [Array<String>] patterns added with {#number_by_content}
43
+ attr_reader :numbered_by_content
44
+
45
+ # @return [Array<String>] patterns added with {#generated_keys}
46
+ attr_reader :generated_key_columns
47
+
48
+ # @return [Hash{String => Array<String>}] patterns added with {#references_to}, by referenced table
49
+ attr_reader :reference_columns
50
+
36
51
  def initialize
37
52
  @tape_library_dir = "spec/db_tapes"
38
53
  @databases = {}
@@ -40,6 +55,11 @@ module TapeDeck
40
55
  @on_schema_change = :error
41
56
  @excluded_tables = []
42
57
  @excluded_columns = []
58
+ @included_views = []
59
+ @kept_new_ids = []
60
+ @numbered_by_content = []
61
+ @generated_key_columns = []
62
+ @reference_columns = {}
43
63
  end
44
64
 
45
65
  # @param mode [Symbol] one of {Tape::RECORD_MODES}
@@ -80,9 +100,72 @@ module TapeDeck
80
100
  @excluded_columns.concat(patterns.map(&:to_s))
81
101
  end
82
102
 
103
+ # Adds matching views to snapshots. Views are left out by default, because the changes of the tables behind them
104
+ # are recorded anyway. Include them to also notice changed view definitions, or data a view reads from tables
105
+ # that are not recorded. Included views are read in full on every snapshot, and compared as a multiset of rows,
106
+ # because they have no primary key. {#exclude_columns} applies to their columns, {#exclude_tables} does not.
107
+ #
108
+ # @param patterns [Array<String>] globs on +"<database>.<view>"+
109
+ # @return [void]
110
+ # @example
111
+ # c.include_views "shop.order_totals", "billing.*"
112
+ def include_views(*patterns)
113
+ @included_views.concat(patterns.map(&:to_s))
114
+ end
115
+
116
+ # Keeps the generated ids of rows inserted into matching tables, instead of replacing them with placeholders.
117
+ # See "New rows" in {TapeDeck}.
118
+ #
119
+ # @param patterns [Array<String>] globs on +"<database>.<table>"+
120
+ # @return [void]
121
+ # @example
122
+ # c.keep_new_ids "shop.audit_log"
123
+ def keep_new_ids(*patterns)
124
+ @kept_new_ids.concat(patterns.map(&:to_s))
125
+ end
126
+
127
+ # Numbers the new rows of matching tables by their content instead of the order they were created in, so creating
128
+ # the same rows in a different order still verifies. For example an import, whose order doesn't matter. Rows with
129
+ # the same content are numbered in creation order. See "New rows" in {TapeDeck}.
130
+ #
131
+ # @param patterns [Array<String>] globs on +"<database>.<table>"+
132
+ # @return [void]
133
+ # @example
134
+ # c.number_by_content "shop.imported_products"
135
+ def number_by_content(*patterns)
136
+ @numbered_by_content.concat(patterns.map(&:to_s))
137
+ end
138
+
139
+ # Marks matching primary keys as generated, like auto increment ones are: keys that the database or the
140
+ # application fills in, for example UUIDs. Their new rows get placeholders, see "New rows" in {TapeDeck}. Keys that
141
+ # are not integers have no creation order, so their rows are numbered by content.
142
+ #
143
+ # @param patterns [Array<String>] globs on +"<database>.<table>.<column>"+
144
+ # @return [void]
145
+ # @example
146
+ # c.generated_keys "shop.tokens.id", "*.uuid"
147
+ def generated_keys(*patterns)
148
+ @generated_key_columns.concat(patterns.map(&:to_s))
149
+ end
150
+
151
+ # Marks matching columns as holding keys of a table, like a declared foreign key does. Needed for references
152
+ # across databases, and ones without a foreign key. Overrides a declared foreign key of the same column.
153
+ #
154
+ # @param table [String] +"<database>.<table>"+ whose keys the columns hold
155
+ # @param patterns [Array<String>] globs on +"<database>.<table>.<column>"+
156
+ # @return [void]
157
+ # @example
158
+ # c.references_to "shop.orders", "billing.invoices.order_id", "*.last_order_id"
159
+ def references_to(table, *patterns)
160
+ (@reference_columns[table.to_s] ||= []).concat(patterns.map(&:to_s))
161
+ end
162
+
83
163
  # Wraps RSpec examples tagged +:tape_deck+ in a tape. The tape is named after the example's description hierarchy
84
164
  # unless +tape_name:+ is given. When the example fails, the tape is neither recorded nor verified.
85
165
  #
166
+ # Before the first example runs, the suite fails if tagged examples would share a tape file, for example the same
167
+ # description in a loop. See {.check_rspec_tape_names}.
168
+ #
86
169
  # @return [void]
87
170
  # @example
88
171
  # it "creates an order", :tape_deck do ... end
@@ -90,11 +173,13 @@ module TapeDeck
90
173
  # it "creates an order", tape_deck: { databases: %w[shop] } do ... end
91
174
  def configure_rspec_metadata!
92
175
  ::RSpec.configure do |config|
176
+ # All examples are loaded by now, so examples sharing a tape by mistake fail the run before anything is recorded
177
+ config.before(:suite) { Configuration.check_rspec_tape_names(::RSpec.world.all_examples) }
178
+
93
179
  config.around(:each, :tape_deck) do |example|
94
- options = Hash.try_convert(example.metadata[:tape_deck]) || {} # a bare :tape_deck tag is `true`
95
- name = options[:tape_name] || Configuration.rspec_tape_name(example)
180
+ name, options = Configuration.rspec_tape(example)
96
181
 
97
- TapeDeck.insert_tape(name, **options.except(:tape_name))
182
+ TapeDeck.insert_tape(name, **options)
98
183
  example.run
99
184
  example_failed = !example.exception.nil?
100
185
  TapeDeck.eject_tape(discard: example_failed)
@@ -102,6 +187,34 @@ module TapeDeck
102
187
  end
103
188
  end
104
189
 
190
+ # @param example [RSpec::Core::Example] an example tagged +:tape_deck+
191
+ # @return [Array(String, Hash)] its tape name, and the options for {TapeDeck.insert_tape}
192
+ # @api private
193
+ def self.rspec_tape(example)
194
+ options = Hash.try_convert(example.metadata[:tape_deck]) || {} # a bare :tape_deck tag is `true`
195
+ name = options[:tape_name] || rspec_tape_name(example)
196
+ [name, options.except(:tape_name)]
197
+ end
198
+
199
+ # Examples that share a tape file would record and verify each other's changes, for example the same description
200
+ # in a loop. Explicit +tape_name:+ options count too.
201
+ #
202
+ # @param examples [Array<RSpec::Core::Example>] all loaded examples
203
+ # @return [void]
204
+ # @raise [Error] if tagged examples share a tape file, listing them
205
+ # @api private
206
+ def self.check_rspec_tape_names(examples)
207
+ tagged = examples.select { _1.metadata[:tape_deck] }
208
+ shared = tagged.group_by { Tape.file_name_for(rspec_tape(_1).first) }.select { |_, group| group.size > 1 }
209
+ return if shared.empty?
210
+
211
+ details = shared.map do |file_name, group|
212
+ lines = group.map { " #{_1.id} #{rspec_tape(_1).first.inspect}" }
213
+ " #{file_name}:\n#{lines.join("\n")}"
214
+ end
215
+ raise Error, "TapeDeck: examples share a tape. Give each its own description or tape_name:\n#{details.join("\n")}"
216
+ end
217
+
105
218
  # @param example [RSpec::Core::Example]
106
219
  # @return [String] the example's description hierarchy, e.g. +"Checkout/creates an order"+
107
220
  # @api private
@@ -43,10 +43,9 @@ module TapeDeck
43
43
  kept.reject { |_, entries| entries.empty? }
44
44
  end
45
45
 
46
- # Rows are sorted again, because their recorded order may depend on values that are now ignored.
47
46
  def filter_rows(table_name, rows)
48
47
  kept = rows.map { |row| filter_columns(table_name, row) }
49
- kept.reject(&:empty?).sort_by(&:inspect)
48
+ kept.reject(&:empty?)
50
49
  end
51
50
 
52
51
  # The "key" of an update is kept as it is, because it identifies the row.
@@ -0,0 +1,168 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TapeDeck
4
+ # Replaces the generated keys of inserted rows with placeholders like +"new shop.orders 1"+, numbered per table, and
5
+ # references to those rows with the same placeholder. So a tape doesn't depend on where the database's counters
6
+ # stand. Ids of rows that existed before stay as they are. See "New rows" in {TapeDeck}.
7
+ #
8
+ # @api private
9
+ class NewRows
10
+ # @param recorder [Recorder] tells the generated keys and references of tables
11
+ # @param keep [Array<String>] patterns on +"<database>.<table>"+ for tables whose new ids are kept, see
12
+ # {Configuration#keep_new_ids}
13
+ # @param by_content [Array<String>] patterns on +"<database>.<table>"+ for tables whose new rows are numbered by
14
+ # content instead of creation order, see {Configuration#number_by_content}
15
+ # @param generated_keys [Array<String>] patterns for more generated keys, see {Configuration#generated_keys}
16
+ # @param references [Hash{String => Array<String>}] more references, see {Configuration#references_to}
17
+ def initialize(recorder, keep: [], by_content: [], generated_keys: [], references: {})
18
+ @recorder = recorder
19
+ @keep = keep
20
+ @by_content = by_content
21
+ @generated_keys = generated_keys
22
+ @references = references
23
+ end
24
+
25
+ # @param recorder [Recorder]
26
+ # @param tape_deck [Hash] the +tape_deck+ section of a tape, see {Tape#read}
27
+ # @return [NewRows] numbering with the rules the tape was recorded with
28
+ def self.recorded_with(recorder, tape_deck)
29
+ new(
30
+ recorder,
31
+ keep: tape_deck.fetch("keep_new_ids"),
32
+ by_content: tape_deck.fetch("number_by_content"),
33
+ generated_keys: tape_deck.fetch("generated_keys"),
34
+ references: tape_deck.fetch("references_to")
35
+ )
36
+ end
37
+
38
+ # @return [Hash] the rules, as stored in the +tape_deck+ section of a tape
39
+ def rules
40
+ {
41
+ "keep_new_ids" => @keep,
42
+ "number_by_content" => @by_content,
43
+ "generated_keys" => @generated_keys,
44
+ "references_to" => @references
45
+ }
46
+ end
47
+
48
+ # @param diff [Hash] a diff as stored on a tape
49
+ # @return [Hash] the diff with placeholders
50
+ def apply(diff)
51
+ keys = @recorder.generated_keys(diff, patterns: @generated_keys).to_h do |db_name, tables|
52
+ [db_name, tables.reject { |table, _| Patterns.match?(@keep, "#{db_name}.#{table}") }]
53
+ end
54
+ references = @recorder.references(diff, patterns: @references)
55
+ columns = diff.to_h do |db_name, tables|
56
+ [db_name, tables.keys.to_h { [_1, placeholder_columns(db_name, _1, keys, references)] }]
57
+ end
58
+ placeholders = number(diff, keys, columns)
59
+
60
+ diff.to_h do |db_name, tables|
61
+ [db_name, tables.to_h { |table, changes| [table, replace(changes, columns.dig(db_name, table), placeholders)] }]
62
+ end
63
+ end
64
+
65
+ private
66
+
67
+ # @return [Hash{String => Array(String, String)}] column -> +[database, table]+ whose new keys it can hold: the
68
+ # table's own key, and its references
69
+ def placeholder_columns(db_name, table, keys, references)
70
+ key = keys.dig(db_name, table)
71
+ own_key = if key
72
+ {key => [db_name, table]}
73
+ else
74
+ {}
75
+ end
76
+ references.dig(db_name, table).to_h.merge(own_key)
77
+ end
78
+
79
+ # Tables numbered by content go after the ones they reference, so their content can hold those placeholders.
80
+ # Keys that are not integers, like UUIDs, tell nothing about creation order, so their tables are numbered by
81
+ # content too.
82
+ #
83
+ # @return [Hash{Array(String, String) => Hash{Object => String}}] +[database, table]+ -> key -> placeholder
84
+ def number(diff, keys, columns)
85
+ tables = keys.flat_map { |db_name, db_keys| db_keys.map { |table, key| [db_name, table, key] } }
86
+ new_keys = tables.to_h do |db_name, table, key|
87
+ [[db_name, table], diff.dig(db_name, table, "inserted").map { _1.fetch(key) }]
88
+ end
89
+ by_content, by_creation = tables.partition do |db_name, table, _|
90
+ content?(db_name, table) || !new_keys.fetch([db_name, table]).all?(Integer)
91
+ end
92
+
93
+ placeholders = {}
94
+ by_creation.each do |db_name, table, key|
95
+ rows = diff.dig(db_name, table, "inserted").sort_by { _1.fetch(key) }
96
+ placeholders[[db_name, table]] = assign(db_name, table, rows.map { _1.fetch(key) })
97
+ end
98
+ pending = by_content
99
+ until pending.empty?
100
+ ready = pending.reject do |db_name, table, _|
101
+ references_pending?(columns.dig(db_name, table), [db_name, table], pending)
102
+ end
103
+ ready = pending if ready.empty? # a cycle, see #content
104
+ ready.each do |db_name, table, key|
105
+ rows = diff.dig(db_name, table, "inserted")
106
+ sorted = rows.sort_by { [content(_1, key, columns.dig(db_name, table), placeholders, new_keys), _1.fetch(key)] }
107
+ placeholders[[db_name, table]] = assign(db_name, table, sorted.map { _1.fetch(key) })
108
+ end
109
+ pending -= ready
110
+ end
111
+ placeholders
112
+ end
113
+
114
+ def content?(db_name, table) = Patterns.match?(@by_content, "#{db_name}.#{table}")
115
+
116
+ def references_pending?(columns, own, pending)
117
+ pending_tables = pending.map { |db_name, table, _| [db_name, table] } - [own]
118
+ columns.values.any? { pending_tables.include?(_1) }
119
+ end
120
+
121
+ # The row without its key, as JSON. A reference to a new row that has no placeholder yet (the table references
122
+ # itself, or tables reference each other) only says which table it points to, because its id is not stable.
123
+ def content(row, key, columns, placeholders, new_keys)
124
+ values = row.except(key).to_h do |column, value|
125
+ target = columns[column]
126
+ replaced = if target.nil?
127
+ value
128
+ elsif placeholders.key?(target)
129
+ placeholders.dig(target, value) || value
130
+ elsif new_keys.fetch(target, []).include?(value)
131
+ "new #{target.join(".")}"
132
+ else
133
+ value
134
+ end
135
+ [column, replaced]
136
+ end
137
+ JSON.generate(values, allow_nan: true)
138
+ end
139
+
140
+ def assign(db_name, table, values)
141
+ values.each_with_index.to_h { |value, i| [value, "new #{db_name}.#{table} #{i + 1}"] }
142
+ end
143
+
144
+ # Rows that existed before keep their own ids, so only inserted rows and updated values can hold new ones.
145
+ #
146
+ # @param columns [Hash{String => Array(String, String)}] column -> +[database, table]+ whose new keys it can hold
147
+ def replace(changes, columns, placeholders)
148
+ swap = ->(column, value) do
149
+ target = columns[column]
150
+ (target && placeholders.dig(target, value)) || value
151
+ end
152
+ changes.to_h do |section, entries|
153
+ replaced = case section
154
+ when "inserted"
155
+ entries.map { |row| row.to_h { |column, value| [column, swap.call(column, value)] } }
156
+ when "updated"
157
+ entries.map do |update|
158
+ values = update.fetch("changes").to_h { |column, pair| [column, pair.map { swap.call(column, _1) }] }
159
+ update.merge("changes" => values)
160
+ end
161
+ else
162
+ entries
163
+ end
164
+ [section, replaced]
165
+ end
166
+ end
167
+ end
168
+ end
@@ -4,7 +4,7 @@ module TapeDeck
4
4
  # Snapshot of one table.
5
5
  #
6
6
  # @!attribute [r] checksum
7
- # @return [Integer, nil] result of +CHECKSUM TABLE+
7
+ # @return [Object, nil] checksum from the adapter, see {Adapters::Base#checksums}
8
8
  # @!attribute [r] pk
9
9
  # @return [Array<String>] primary key columns, empty if the table has none
10
10
  # @!attribute [r] rows
@@ -13,26 +13,28 @@ module TapeDeck
13
13
  # @api private
14
14
  TableSnap = Data.define(:checksum, :pk, :rows)
15
15
 
16
- # Takes snapshots of all configured databases and diffs them. Used by {Session}, but usable on its own when you don't
17
- # need tape files.
16
+ # Takes snapshots of all configured databases and diffs them. The database specific parts are in {Adapters}. Used by
17
+ # {Session}, but usable on its own when you don't need tape files.
18
18
  class Recorder
19
- # Database types of TIME columns, with or without fractional seconds. Sequel only types +time+ as +:time+, not
20
- # +time(6)+.
21
- TIME_DB_TYPE = /\Atime\b/i
22
-
23
19
  # @return [Array<String>] see {Configuration#exclude_tables}
24
20
  attr_reader :excluded_tables
25
21
 
26
22
  # @return [Array<String>] see {Configuration#exclude_columns}
27
23
  attr_reader :excluded_columns
28
24
 
25
+ # @return [Array<String>] see {Configuration#include_views}
26
+ attr_reader :included_views
27
+
29
28
  # @param dbs [Hash{String => Sequel::Database}]
30
29
  # @param exclude_tables [Array<String>] see {Configuration#exclude_tables}
31
30
  # @param exclude_columns [Array<String>] see {Configuration#exclude_columns}
32
- def initialize(dbs, exclude_tables: [], exclude_columns: [])
31
+ # @param include_views [Array<String>] see {Configuration#include_views}
32
+ def initialize(dbs, exclude_tables: [], exclude_columns: [], include_views: [])
33
33
  @dbs = dbs
34
+ @adapters = dbs.transform_values { Adapters.for(_1) }
34
35
  @excluded_tables = exclude_tables
35
36
  @excluded_columns = exclude_columns
37
+ @included_views = include_views
36
38
  end
37
39
 
38
40
  # @yield the action whose database changes are recorded
@@ -43,8 +45,9 @@ module TapeDeck
43
45
  diff(before, snapshot(previous: before))
44
46
  end
45
47
 
46
- # Reads all tables that are not excluded. With +previous:+, tables whose +CHECKSUM TABLE+ is unchanged reuse the
47
- # previous snapshot, so only changed tables are read again.
48
+ # Reads all tables that are not excluded, and all included views. With +previous:+, tables whose checksum is
49
+ # unchanged reuse the previous snapshot, so only changed tables are read again. Views and tables without a
50
+ # checksum are always read again.
48
51
  #
49
52
  # @param previous [Hash, nil] an earlier result of this method
50
53
  # @return [Hash{String => Hash{String => TableSnap}}]
@@ -60,7 +63,8 @@ module TapeDeck
60
63
  end
61
64
  [table, snap]
62
65
  end
63
- [name, tables]
66
+ views = view_names(db, name).to_h { [_1, dump(db, name, _1, nil, view: true)] }
67
+ [name, tables.merge(views)]
64
68
  end
65
69
  end
66
70
 
@@ -77,25 +81,78 @@ module TapeDeck
77
81
  end
78
82
  end
79
83
 
80
- # What exclude rules leave out of each database, by default the ones this recorder was created with. Reads the
81
- # current schema, so other rules show what they would have left out.
84
+ # Applies recording rules to the current schema of each database, by default the rules this recorder was created
85
+ # with. Other rules show what they would leave out and include.
82
86
  #
83
87
  # @param exclude_tables [Array<String>] table patterns, see {Configuration#exclude_tables}
84
88
  # @param exclude_columns [Array<String>] column patterns, see {Configuration#exclude_columns}
85
- # @return [Hash{String => Array<String>}] database -> sorted +"<table>"+ and +"<table>.<column>"+ names
86
- def exclusions(exclude_tables: @excluded_tables, exclude_columns: @excluded_columns)
89
+ # @param include_views [Array<String>] view patterns, see {Configuration#include_views}
90
+ # @return [Hash{String => Hash{String => Array<String>}}] database -> +"excluded"+: sorted +"<table>"+ and
91
+ # +"<table>.<column>"+ names the exclude rules leave out, +"views"+: sorted names of the included views
92
+ def resolve(exclude_tables: @excluded_tables, exclude_columns: @excluded_columns, include_views: @included_views)
87
93
  @dbs.to_h do |db_name, db|
88
94
  tables = db.tables.map(&:to_s)
95
+ views = view_names(db, db_name, include_views)
89
96
  table_names = tables.select { Patterns.match?(exclude_tables, "#{db_name}.#{_1}") }
90
- column_names = (tables - table_names).flat_map do |table|
97
+ column_names = (tables - table_names + views).flat_map do |table|
91
98
  excluded_column_names(db_name, table, db.schema(table.to_sym), exclude_columns).map { "#{table}.#{_1}" }
92
99
  end
93
- [db_name, (table_names + column_names).sort]
100
+ [db_name, {"excluded" => (table_names + column_names).sort, "views" => views}]
101
+ end
102
+ end
103
+
104
+ # Generated keys of the tables the diff inserts rows into: a single primary key column that is auto increment,
105
+ # or matches one of the patterns. Views have none.
106
+ #
107
+ # @param diff [Hash] result of {#diff}
108
+ # @param patterns [Array<String>] globs on +"<database>.<table>.<column>"+, see {Configuration#generated_keys}
109
+ # @return [Hash{String => Hash{String => String}}] database -> table -> key column
110
+ def generated_keys(diff, patterns: [])
111
+ diff.to_h do |db_name, tables|
112
+ db = @dbs.fetch(db_name)
113
+ real_tables = db.tables.map(&:to_s)
114
+ keys = tables.filter_map do |table, changes|
115
+ next unless changes.key?("inserted") && real_tables.include?(table)
116
+
117
+ pk = db.schema(table.to_sym).select { |_, info| info[:primary_key] }
118
+ next unless pk.size == 1
119
+
120
+ column, info = pk.first
121
+ generated = info[:auto_increment] || Patterns.match?(patterns, "#{db_name}.#{table}.#{column}")
122
+ [table, column.to_s] if generated
123
+ end
124
+ [db_name, keys.to_h]
125
+ end
126
+ end
127
+
128
+ # Single column references of the tables in the diff: declared foreign keys, and columns that match the patterns.
129
+ # Matching patterns win over declared foreign keys.
130
+ #
131
+ # @param diff [Hash] result of {#diff}
132
+ # @param patterns [Hash{String => Array<String>}] +"<database>.<table>"+ -> globs on +"<database>.<table>.<column>"+
133
+ # of columns holding its keys, see {Configuration#references_to}
134
+ # @return [Hash{String => Hash{String => Hash{String => Array(String, String)}}}] database -> table -> column ->
135
+ # +[database, table]+ it references
136
+ def references(diff, patterns: {})
137
+ diff.to_h do |db_name, tables|
138
+ db = @dbs.fetch(db_name)
139
+ real_tables = db.tables.map(&:to_s)
140
+ references = tables.keys.select { real_tables.include?(_1) }.to_h do |table|
141
+ declared = db.foreign_key_list(table.to_sym).filter_map do |foreign_key|
142
+ [foreign_key[:columns].first.to_s, [db_name, foreign_key[:table].to_s]] if foreign_key[:columns].size == 1
143
+ end
144
+ configured = db.schema(table.to_sym).filter_map do |column, _|
145
+ target = patterns.find { |_, columns| Patterns.match?(columns, "#{db_name}.#{table}.#{column}") }&.first
146
+ [column.to_s, target.split(".", 2)] if target
147
+ end
148
+ [table, declared.to_h.merge(configured.to_h)]
149
+ end
150
+ [db_name, references]
94
151
  end
95
152
  end
96
153
 
97
154
  # @param db_name [String]
98
- # @param name [String] +"<table>"+ or +"<table>.<column>"+, as returned by {#exclusions}
155
+ # @param name [String] +"<table>"+ or +"<table>.<column>"+, as returned by {#resolve}
99
156
  # @return [Boolean] whether the exclude rules leave this table or column out. Says nothing about primary keys,
100
157
  # which are never left out.
101
158
  def excluded?(db_name, name)
@@ -118,6 +175,13 @@ module TapeDeck
118
175
  end
119
176
  end
120
177
 
178
+ # @return [Array<String>] sorted names of the views the patterns include
179
+ def view_names(db, db_name, patterns = @included_views)
180
+ return [] if patterns.empty?
181
+
182
+ db.views.map(&:to_s).select { Patterns.match?(patterns, "#{db_name}.#{_1}") }.sort
183
+ end
184
+
121
185
  # Diffs one database. A table missing on one side (created or dropped by the action) counts as empty there.
122
186
  def tables_diff(before, after)
123
187
  (before.keys | after.keys).sort.each_with_object({}) do |table, out|
@@ -129,23 +193,24 @@ module TapeDeck
129
193
  end
130
194
  end
131
195
 
132
- # @return [Hash{String => Integer, nil}] table name -> checksum
196
+ # @return [Hash{String => Object, nil}] table name -> checksum, for the tables that are not excluded
133
197
  def checksums(db, name)
134
198
  tables = db.tables.map(&:to_s).sort.reject { excluded?(name, _1) }
135
- return {} if tables.empty?
136
-
137
- list = tables.map { db.literal(Sequel.identifier(_1)) }.join(", ")
138
- db.fetch("CHECKSUM TABLE #{list}").to_h do |r|
139
- _db_name, table = r[:Table].split(".", 2) # MySQL reports "<db>.<table>"
140
- [table, r[:Checksum]]
141
- end
199
+ @adapters.fetch(name).checksums(tables)
142
200
  end
143
201
 
202
+ # Views have no primary key. Whatever the schema says, their rows are compared as a multiset.
203
+ #
144
204
  # @return [TableSnap]
145
- def dump(db, name, table, sum)
205
+ def dump(db, name, table, sum, view: false)
206
+ adapter = @adapters.fetch(name)
146
207
  schema = db.schema(table.to_sym)
147
- pk = schema.select { |_, i| i[:primary_key] }.map(&:first)
148
- types = schema.to_h { |col, i| [col, column_type(i)] }
208
+ pk = if view
209
+ []
210
+ else
211
+ schema.select { |_, i| i[:primary_key] }.map(&:first)
212
+ end
213
+ columns = schema.to_h
149
214
 
150
215
  drop = excluded_column_names(name, table, schema)
151
216
 
@@ -154,26 +219,16 @@ module TapeDeck
154
219
 
155
220
  rows = {}
156
221
  ds.each do |r|
157
- row = r.except(*drop).to_h { |col, v| [col.to_s, serialize(v, types[col])] }
222
+ row = r.except(*drop).to_h { |col, v| [col.to_s, adapter.serialize(v, columns.fetch(col, {}))] }
158
223
  if pk.empty?
159
224
  rows[row] = rows.fetch(row, 0) + 1
160
225
  else
161
- rows[pk.map { serialize(r[_1], types[_1]) }] = row
226
+ rows[pk.map { adapter.serialize(r[_1], columns[_1]) }] = row
162
227
  end
163
228
  end
164
229
  TableSnap.new(checksum: sum, pk: pk.map(&:to_s), rows: rows)
165
230
  end
166
231
 
167
- # @param info [Hash] column info from +db.schema+
168
- # @return [Symbol, nil] Sequel's column type, +:time+ for all TIME columns, see {TIME_DB_TYPE}
169
- def column_type(info)
170
- if TIME_DB_TYPE.match?(info[:db_type].to_s)
171
- :time
172
- else
173
- info[:type]
174
- end
175
- end
176
-
177
232
  def empty_like(snap) = TableSnap.new(checksum: nil, pk: snap.pk, rows: {})
178
233
 
179
234
  def table_diff(b, a)
@@ -216,55 +271,7 @@ module TapeDeck
216
271
  compact("inserted" => sort_rows(inserted), "deleted" => sort_rows(deleted))
217
272
  end
218
273
 
219
- def sort_rows(rows) = rows.sort_by(&:inspect)
274
+ def sort_rows(rows) = Rows.sort(rows)
220
275
  def compact(h) = h.reject { |_, v| v.empty? }
221
-
222
- # Converts a column value to a YAML-stable scalar. +type+ is the column type from +db.schema+.
223
- def serialize(v, type = nil)
224
- case v
225
- when String then maybe_utf8(v)
226
- when BigDecimal then v.to_s("F")
227
- when Sequel::SQLTime then time_of_day(v)
228
- when Time then timestamp(v, type)
229
- when Date then v.iso8601
230
- else v # Integer, Float, true/false, nil
231
- end
232
- end
233
-
234
- # mysql2 returns TIME columns as a +Time+ on 2000-01-01, not as +Sequel::SQLTime+. Those keep their local time of
235
- # day, because converting them to UTC would shift the hour.
236
- def timestamp(v, type)
237
- if type == :time
238
- time_of_day(v)
239
- else
240
- v.utc.iso8601(6)
241
- end
242
- end
243
-
244
- def time_of_day(v) = v.strftime("%H:%M:%S.%6N")
245
-
246
- # Small binaries encode as hex. Large ones are shown as a digest so the YAML stays small.
247
- def bin(v)
248
- if v.bytesize <= 64
249
- "0x#{v.unpack1("H*")}"
250
- else
251
- "sha256:#{Digest::SHA256.hexdigest(v)}"
252
- end
253
- end
254
-
255
- # BINARY, VARBINARY and BLOB columns arrive as ASCII-8BIT strings (or Sequel::SQL::Blob, depending on the adapter).
256
- # Ones that hold valid UTF-8 become plain UTF-8 strings, because YAML would otherwise store them as base64
257
- # (+!binary+). Everything else goes through {#bin}.
258
- def maybe_utf8(v)
259
- binary = v.encoding == Encoding::BINARY || v.is_a?(Sequel::SQL::Blob)
260
- return v unless binary
261
-
262
- utf8 = String.new(v, encoding: Encoding::UTF_8)
263
- if utf8.valid_encoding?
264
- utf8
265
- else
266
- bin(v)
267
- end
268
- end
269
276
  end
270
277
  end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TapeDeck
4
+ # A stable order for rows, so tapes come out the same on every Ruby version. Rows hold plain values only, see
5
+ # {Adapters::Base#serialize}, and the JSON format pins down how those are written. +inspect+ does not, it changed
6
+ # in Ruby 3.4.
7
+ #
8
+ # @api private
9
+ module Rows
10
+ # @param rows [Array<Hash>]
11
+ # @return [Array<Hash>] sorted by their JSON
12
+ def self.sort(rows)
13
+ rows.sort_by { JSON.generate(_1, allow_nan: true) }
14
+ end
15
+ end
16
+ end