tape_deck 0.1.0.beta1 → 0.1.0

data/lib/tape_deck/session.rb

@@ -13,11 +13,14 @@ module TapeDeck
     # @param recorder [Recorder]
     # @param filter [Filter] narrows what is compared when verifying
     # @param on_schema_change [Symbol] see {Configuration#on_schema_change}
-    def initialize(tape, recorder, filter, on_schema_change: :error)
+    # @param new_rows [NewRows] replaces the ids of new rows with placeholders when recording. Verifying uses the rules
+    #   the tape was recorded with.
+    def initialize(tape, recorder, filter, on_schema_change: :error, new_rows: NewRows.new(recorder))
       @tape = tape
       @recorder = recorder
       @filter = filter
       @on_schema_change = on_schema_change
+      @new_rows = new_rows
       @before = recorder.snapshot
     end
 
@@ -31,16 +34,20 @@ module TapeDeck
       return if discard
 
       after = @recorder.snapshot(previous: @before)
-      actual = round_trip(@recorder.diff(@before, after))
-      exclusions = @recorder.exclusions
+      diff = round_trip(@recorder.diff(@before, after))
+      current = @recorder.resolve
       if tape.recording?
-        tape.write(actual, recorded: {
+        tape.write(@new_rows.apply(diff), tape_deck: {
           "exclude_tables" => @recorder.excluded_tables,
           "exclude_columns" => @recorder.excluded_columns,
-          "databases" => exclusions
+          "include_views" => @recorder.included_views,
+          **@new_rows.rules,
+          "databases" => current
         })
       else
-        verify(actual, exclusions)
+        # Numbering only changes how ids are written, so the tape's own rules apply, whatever the configuration says now
+        actual = NewRows.recorded_with(@recorder, tape.read.fetch("tape_deck")).apply(diff)
+        verify(actual, current)
       end
     end
 
@@ -49,34 +56,50 @@ module TapeDeck
     # @return [Hash] the diff with the same shape as a tape read from disk
     def round_trip(diff) = YAML.safe_load(Tape.serialize(diff))
 
-    # Compares the watched databases only. What is excluded now, but is on the tape, is dropped from both sides.
-    def verify(actual, exclusions)
-      recorded = tape.read.fetch("recorded")
-      tape_exclusions = recorded.fetch("databases").slice(*exclusions.keys)
-      report_schema_changes(recorded, tape_exclusions)
-      filter = @filter.ignoring(newly_excluded(tape_exclusions, exclusions))
-      expected = filter.apply(tape.read.fetch("diff").slice(*exclusions.keys))
-      actual = filter.apply(actual)
+    # Compares the watched databases only. What the current rules leave out, but is on the tape, is dropped from both
+    # sides. So are views a schema change added.
+    #
+    # @param current [Hash] what the current rules resolve to, see {Recorder#resolve}
+    def verify(actual, current)
+      recorded = tape.read.fetch("tape_deck")
+      on_tape = recorded.fetch("databases").slice(*current.keys)
+      by_recorded_rules = @recorder.resolve(
+        exclude_tables: recorded.fetch("exclude_tables"),
+        exclude_columns: recorded.fetch("exclude_columns"),
+        include_views: recorded.fetch("include_views")
+      )
+      report_schema_changes(on_tape, by_recorded_rules)
+      ignored = newly_excluded(on_tape, current) + changed_views(on_tape, current, by_recorded_rules)
+      filter = @filter.ignoring(ignored)
+      expected = in_stable_order(filter.apply(tape.read.fetch("diff").slice(*current.keys)))
+      actual = in_stable_order(filter.apply(actual))
       raise UnexpectedDiffError.new(tape, expected, actual) unless expected == actual
     end
 
-    # The recorded exclude rules, applied to the current schema, must leave out what they did when recording.
-    def report_schema_changes(recorded, tape_exclusions)
+    # The order of rows on a tape is for reading only, so it must not decide whether the tape matches. Filtered rows
+    # need it too, because their order may have depended on values that are ignored now.
+    def in_stable_order(diff)
+      diff.transform_values do |tables|
+        tables.transform_values { |changes| changes.transform_values { Rows.sort(_1) } }
+      end
+    end
+
+    # The recorded rules, applied to the current schema, must leave out and include what they did when recording.
+    def report_schema_changes(on_tape, by_recorded_rules)
       return if @on_schema_change == :ignore
 
-      now = @recorder.exclusions(
-        exclude_tables: recorded.fetch("exclude_tables"),
-        exclude_columns: recorded.fetch("exclude_columns")
-      )
-      changes = tape_exclusions.filter_map do |db_name, names|
-        added = now.fetch(db_name) - names
-        removed = names - now.fetch(db_name)
-        "#{db_name}: added #{added.inspect}, removed #{removed.inspect}" unless added.empty? && removed.empty?
+      changes = on_tape.flat_map do |db_name, lists|
+        Tape::DATABASE_LISTS.filter_map do |list|
+          now = by_recorded_rules.fetch(db_name).fetch(list)
+          added = now - lists.fetch(list)
+          removed = lists.fetch(list) - now
+          "#{db_name} #{list}: added #{added.inspect}, removed #{removed.inspect}" unless added.empty? && removed.empty?
+        end
       end
       return if changes.empty?
 
-      message = "schema changed since tape #{tape.file} was recorded, its exclude rules now match " \
-        "other tables or columns (#{changes.join(", ")})"
+      message = "schema changed since tape #{tape.file} was recorded, its recording rules now match " \
+        "other tables, columns or views (#{changes.join(", ")})"
       raise IncompatibleTapeError, message if @on_schema_change == :error
 
       warn "TapeDeck: #{message}"
@@ -84,16 +107,36 @@ module TapeDeck
 
     # @return [Array<String>] patterns for what is excluded now, but was recorded on the tape
     # @raise [IncompatibleTapeError] if the tape left out something that is no longer excluded
-    def newly_excluded(tape_exclusions, exclusions)
-      exclusions.flat_map do |db_name, names|
-        tape_names = tape_exclusions.fetch(db_name)
+    def newly_excluded(on_tape, current)
+      current.flat_map do |db_name, lists|
+        tape_names = on_tape.fetch(db_name).fetch("excluded")
         no_longer_excluded = tape_names.reject { @recorder.excluded?(db_name, _1) }
         unless no_longer_excluded.empty?
           raise IncompatibleTapeError, "tape #{tape.file} left out #{no_longer_excluded.inspect} of #{db_name}, " \
             "which is no longer excluded. Re-record the tape."
         end
 
-        (names - tape_names).map { Patterns.escape("#{db_name}.#{_1}") }
+        (lists.fetch("excluded") - tape_names).map { Patterns.escape("#{db_name}.#{_1}") }
+      end
+    end
+
+    # A view on the tape that is no longer included is not compared. Neither is one included now because a schema
+    # change added it, which {#report_schema_changes} reports.
+    #
+    # @return [Array<String>] patterns for the views not to compare
+    # @raise [IncompatibleTapeError] if a view is included now by changed rules, so the tape lacks it
+    def changed_views(on_tape, current, by_recorded_rules)
+      current.flat_map do |db_name, lists|
+        tape_views = on_tape.fetch(db_name).fetch("views")
+        added = lists.fetch("views") - tape_views
+        newly_included = added - by_recorded_rules.fetch(db_name).fetch("views")
+        unless newly_included.empty?
+          raise IncompatibleTapeError, "tape #{tape.file} has no recording of the views #{newly_included.inspect} " \
+            "of #{db_name}, which are included now. Re-record the tape."
+        end
+
+        no_longer_included = tape_views - lists.fetch("views")
+        (no_longer_included + added).map { Patterns.escape("#{db_name}.#{_1}") }
       end
     end
   end

data/lib/tape_deck/tape.rb

@@ -4,23 +4,33 @@ module TapeDeck
   # One recorded diff, stored as a YAML file. Like a +File+, a tape knows where it lives, whether it is there to be
   # recorded or verified, and how to read and write itself. It never touches a database.
   #
-  # Next to the diff, a tape stores what was recorded: the exclude rules, the watched databases, and per database the
-  # tables and columns the rules left out. Empty lists are left out of the file.
+  # Next to the diff, a tape stores everything that shaped it: the TapeDeck version that recorded it, the recording
+  # rules, the watched databases, and per database the tables and columns the rules left out and the views they
+  # included. Empty lists are left out of the file.
   #
-  #   recorded:
+  #   tape_deck:
+  #     version: 0.1.0
   #     exclude_tables: [shop.sessions]
   #     exclude_columns: ["*.created_at"]
+  #     include_views: [shop.order_totals]
   #     databases:
   #       shop:
   #         excluded: [orders.created_at, sessions]
+  #         views: [order_totals]
   #       billing:
   #   diff:
   #     shop:
   #       orders:
   #         inserted: [...]
   class Tape
-    # Lists in the +recorded+ section of a tape file, written on one line.
-    RECORDED_LISTS = %w[exclude_tables exclude_columns excluded].freeze
+    # Recording rules at the top of the +tape_deck+ section that are lists of patterns.
+    RULES = %w[exclude_tables exclude_columns include_views keep_new_ids number_by_content generated_keys].freeze
+
+    # Recording rules at the top of the +tape_deck+ section that map a table to patterns.
+    TABLE_RULES = %w[references_to].freeze
+
+    # Lists per database in the +tape_deck+ section.
+    DATABASE_LISTS = %w[excluded views].freeze
 
     # Supported values for the +:record+ option:
     # [+:once+]      record if the tape file is missing, otherwise verify
@@ -50,21 +60,16 @@ module TapeDeck
       raise ArgumentError, "invalid record mode #{mode.inspect}, use one of #{RECORD_MODES.inspect}"
     end
 
-    # Like {.serialize}, but writes the {RECORDED_LISTS} on one line, so the +recorded+ section stays short.
+    # Like {.serialize}, but writes the lists of the +tape_deck+ section on one line, so it stays short.
     #
     # @param content [Hash] see {#read}
     # @return [String] the YAML stored in a tape file
     def self.dump(content)
       tree = Psych::Visitors::YAMLTree.create(line_width: -1)
       tree << content
-      tree.tree.each do |node|
-        next unless node.is_a?(Psych::Nodes::Mapping)
-
-        node.children.each_slice(2) do |key, value|
-          recorded_list = RECORDED_LISTS.include?(key.value) && value.is_a?(Psych::Nodes::Sequence)
-          value.style = Psych::Nodes::Sequence::FLOW if recorded_list
-        end
-      end
+      root = tree.tree.children.first.children.first
+      section = root.children.each_slice(2).find { |key, _| key.value == "tape_deck" }&.last
+      section&.each { _1.style = Psych::Nodes::Sequence::FLOW if _1.is_a?(Psych::Nodes::Sequence) }
       tree.tree.yaml(nil, line_width: -1)
     end
 
@@ -95,29 +100,37 @@ module TapeDeck
     end
 
     # @return [String] the tape name with {UNSAFE_FILE_NAME_CHARS} replaced, plus +.yml+
-    def file_name
-      "#{name.gsub(UNSAFE_FILE_NAME_CHARS, "_")}.yml"
-    end
+    def file_name = self.class.file_name_for(name)
+
+    # Different tape names can share a file name, e.g. +"an order"+ and +"an_order"+.
+    #
+    # @param name [String] tape name
+    # @return [String] the name with {UNSAFE_FILE_NAME_CHARS} replaced, plus +.yml+
+    def self.file_name_for(name) = "#{name.to_s.gsub(UNSAFE_FILE_NAME_CHARS, "_")}.yml"
 
     # Reads the tape file once, later calls return the same content.
     #
-    # @return [Hash] +"recorded"+: see {#write}, and +"diff"+: the stored diff
+    # @return [Hash] +"tape_deck"+: see {#write}, and +"diff"+: the stored diff
     # @raise [TapeLoadingError] if the file is not valid YAML or does not hold a recording
     def read
       @content ||= load
     end
 
+    # Also stores the version of TapeDeck that recorded the tape.
+    #
     # @param diff [Hash] the diff to store in the tape file, replacing its content
-    # @param recorded [Hash] +"exclude_tables"+ and +"exclude_columns"+: the exclude rules, +"databases"+: database ->
-    #   names the rules left out of it, as returned by {Recorder#exclusions}
+    # @param tape_deck [Hash] the {RULES} and {TABLE_RULES}, and +"databases"+: what they resolved to in each
+    #   database, as returned by {Recorder#resolve}
     # @return [void]
-    def write(diff, recorded:)
-      databases = recorded.fetch("databases").transform_values do |names|
-        {"excluded" => names} unless names.empty?
+    def write(diff, tape_deck:)
+      databases = tape_deck.fetch("databases").transform_values do |lists|
+        non_empty = lists.slice(*DATABASE_LISTS).reject { |_, list| list.empty? }
+        non_empty unless non_empty.empty?
       end
-      patterns = recorded.slice("exclude_tables", "exclude_columns").reject { |_, list| list.empty? }
+      rules = tape_deck.slice(*RULES, *TABLE_RULES).reject { |_, rule| rule.empty? }
+      content = {"tape_deck" => {"version" => VERSION, **rules, "databases" => databases}, "diff" => diff}
       FileUtils.mkdir_p(File.dirname(file))
-      File.write(file, self.class.dump("recorded" => {**patterns, "databases" => databases}, "diff" => diff))
+      File.write(file, self.class.dump(content))
       @content = nil
     end
 
@@ -127,14 +140,14 @@ module TapeDeck
       content = YAML.safe_load_file(file)
       raise TapeLoadingError, "tape #{file} does not hold a recording" unless valid_content?(content)
 
-      recorded = content["recorded"]
-      databases = recorded["databases"].transform_values { Array(_1&.dig("excluded")) }
+      section = content["tape_deck"]
+      databases = section["databases"].transform_values do |lists|
+        DATABASE_LISTS.to_h { [_1, Array(lists&.dig(_1))] }
+      end
+      rules = RULES.to_h { [_1, Array(section[_1])] }
+      table_rules = TABLE_RULES.to_h { [_1, section[_1].to_h.transform_values { |patterns| Array(patterns) }] }
       {
-        "recorded" => {
-          "exclude_tables" => Array(recorded["exclude_tables"]),
-          "exclude_columns" => Array(recorded["exclude_columns"]),
-          "databases" => databases
-        },
+        "tape_deck" => {"version" => section["version"], **rules, **table_rules, "databases" => databases},
         "diff" => content["diff"]
       }
     rescue Psych::Exception => e
@@ -144,8 +157,8 @@ module TapeDeck
     def valid_content?(content)
       return false unless content.is_a?(Hash) && content["diff"].is_a?(Hash)
 
-      recorded = content["recorded"]
-      databases = recorded["databases"] if recorded.is_a?(Hash)
+      section = content["tape_deck"]
+      databases = section["databases"] if section.is_a?(Hash)
       databases.is_a?(Hash) && databases.values.all? { _1.nil? || _1.is_a?(Hash) }
     end
   end

data/lib/tape_deck/version.rb

@@ -2,5 +2,5 @@
 
 module TapeDeck
   # Version of the gem.
-  VERSION = "0.1.0.beta1"
+  VERSION = "0.1.0"
 end

data/lib/tape_deck.rb

@@ -6,6 +6,7 @@ require "date"
 require "time"
 require "digest"
 require "yaml"
+require "json"
 require "fileutils"
 
 require_relative "tape_deck/version"
@@ -13,11 +14,14 @@ require_relative "tape_deck/errors"
 require_relative "tape_deck/configuration"
 require_relative "tape_deck/tape"
 require_relative "tape_deck/patterns"
+require_relative "tape_deck/rows"
+require_relative "tape_deck/new_rows"
 require_relative "tape_deck/filter"
 require_relative "tape_deck/session"
+require_relative "tape_deck/adapters"
 require_relative "tape_deck/recorder"
 
-# Records what a block of code changes in several MySQL databases, and checks later runs against that recording.
+# Records what a block of code changes in several databases, and checks later runs against that recording.
 # API, naming and flow inspired by the VCR gem.
 #
 # == Tapes
@@ -30,15 +34,19 @@ require_relative "tape_deck/recorder"
 # statements that caused it or their order, so INSERT/UPDATE/DELETE statements generated from it can go wrong in many
 # ways: foreign key constraints, generated columns, triggers.
 #
-# Next to the diff, a tape stores what was recorded: the exclude rules, the watched databases, and the tables and
-# columns the rules left out of each.
+# Next to the diff, a tape stores everything that shaped it: the TapeDeck version that recorded it, the recording
+# rules, the watched databases, and per database the tables and columns the rules left out and the views they
+# included. Empty lists are left out of the file.
 #
-#   recorded:
+#   tape_deck:
+#     version: 0.1.0
 #     exclude_tables: [shop.sessions]
 #     exclude_columns: ["*.created_at"]
+#     include_views: [shop.order_totals]
 #     databases:
 #       shop:
 #         excluded: [orders.created_at, sessions]
+#         views: [order_totals]
 #       billing:
 #   diff:
 #     shop:
@@ -92,6 +100,45 @@ require_relative "tape_deck/recorder"
 # matches dots), so +"*.created_at"+ matches that column in every table of every database, and +"shop.*"+ matches
 # everything in +shop+.
 #
+# == New rows
+#
+# The ids a database hands out depend on where its counters stand, and those move with every insert, also with ones
+# that were rolled back. So a tape doesn't store the generated keys of inserted rows. It numbers them per table in the
+# order the rows were created, and stores placeholders like +"new shop.orders 1"+. Declared foreign keys to those rows
+# get the same placeholder, in inserted rows and in updated values. Ids of rows that existed before stay as they are.
+#
+#   orders:
+#     inserted:
+#     - id: new shop.orders 1
+#   items:
+#     inserted:
+#     - id: new shop.items 1
+#       order_id: new shop.orders 1
+#
+# A key is generated if it is the single, auto increment primary key of its table. Rows are numbered in creation
+# order, so creating the same rows in a different order fails verification. Where that order doesn't matter, like an
+# import, {Configuration#number_by_content} numbers a table's rows by their content instead.
+# {Configuration#keep_new_ids} keeps the real ids of some tables.
+#
+# References across databases, and ones without a foreign key, need {Configuration#references_to}. Keys that are
+# generated, but not auto increment, like UUIDs, need {Configuration#generated_keys}. Those have no creation order, so
+# their rows are numbered by content.
+#
+# The numbering rules are stored on the tape, and verifying uses them, not the current configuration. Numbering only
+# changes how ids are written, so a tape keeps verifying the way it was recorded. New rules apply to tapes recorded
+# afterwards.
+#
+# == Views
+#
+# Views are left out by default. A view holds no data of its own, so the changes of the tables behind it are already
+# on the tape. {Configuration#include_views} adds views to snapshots, to also notice a changed view definition, or data
+# a view reads from tables that are not recorded. Included views are read in full on every snapshot, and compared as a
+# multiset of rows (+inserted+/+deleted+ only), because they have no primary key.
+#
+# Like the exclude rules, the tape stores the include rules and the views they matched. When verifying, a view on the
+# tape that is no longer included is not compared. A view that is included now, but not on the tape, raises
+# {IncompatibleTapeError}, unless the recorded rules match it too: then a schema change added it, see below.
+#
 # == Excluding and ignoring
 #
 # There are two ways to leave data out, and they act at different moments.
@@ -103,7 +150,8 @@ require_relative "tape_deck/recorder"
 # {IncompatibleTapeError} and the tape needs re-recording.
 #
 # The stored rules also reveal schema changes. When verifying, they are applied to the current schema again. If they
-# leave out other tables or columns than when recording, a table or column matching them was added or removed.
+# leave out other tables or columns, or include other views than when recording, a matching table, column or view was
+# added or removed.
 # {Configuration#on_schema_change} decides whether that raises {IncompatibleTapeError}, warns, or is ignored.
 #
 # *Ignore* means recorded, but not compared. A tape always holds everything that is not excluded. When verifying, one
@@ -122,6 +170,7 @@ require_relative "tape_deck/recorder"
 #     c.default_record_mode = :once
 #     c.exclude_tables  "shop.sessions", "*.schema_migrations"
 #     c.exclude_columns "*.created_at", "*.updated_at", "shop.users.last_seen_at"
+#     c.include_views "shop.order_totals"
 #     c.on_schema_change = :warn
 #     c.configure_rspec_metadata!
 #   end
@@ -224,10 +273,18 @@ module TapeDeck
       recorder = Recorder.new(
         select_databases(tape_databases(tape, databases)),
         exclude_tables: configuration.excluded_tables,
-        exclude_columns: configuration.excluded_columns
+        exclude_columns: configuration.excluded_columns,
+        include_views: configuration.included_views
       )
       filter = Filter.new(only: only, ignore: ignore)
-      sessions.push(Session.new(tape, recorder, filter, on_schema_change: configuration.on_schema_change))
+      new_rows = NewRows.new(
+        recorder,
+        keep: configuration.kept_new_ids,
+        by_content: configuration.numbered_by_content,
+        generated_keys: configuration.generated_key_columns,
+        references: configuration.reference_columns
+      )
+      sessions.push(Session.new(tape, recorder, filter, on_schema_change: configuration.on_schema_change, new_rows: new_rows))
       tape
     end
 
@@ -266,7 +323,7 @@ module TapeDeck
     def tape_databases(tape, names)
       return names if tape.recording?
 
-      recorded = tape.read.fetch("recorded").fetch("databases").keys
+      recorded = tape.read.fetch("tape_deck").fetch("databases").keys
       names = Array(names || recorded).map(&:to_s)
       missing = names - recorded
       unless missing.empty?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tape_deck
 version: !ruby/object:Gem::Version
-  version: 0.1.0.beta1
+  version: 0.1.0
 platform: ruby
 authors:
 - Robert Schulze
@@ -40,7 +40,7 @@ dependencies:
 description: Snapshots your databases around an action and stores the row-level diff
   as a YAML tape. Later runs of the same action are verified against the tape, so
   you notice when a change or update alters what gets written. VCR-style API, works
-  with Sequel and MySQL.
+  with Sequel and MySQL, MariaDB, PostgreSQL or SQLite.
 email:
 - robert@dotless.de
 executables: []
@@ -57,11 +57,18 @@ files:
 - README.md
 - Rakefile
 - lib/tape_deck.rb
+- lib/tape_deck/adapters.rb
+- lib/tape_deck/adapters/base.rb
+- lib/tape_deck/adapters/mysql.rb
+- lib/tape_deck/adapters/postgres.rb
+- lib/tape_deck/adapters/sqlite.rb
 - lib/tape_deck/configuration.rb
 - lib/tape_deck/errors.rb
 - lib/tape_deck/filter.rb
+- lib/tape_deck/new_rows.rb
 - lib/tape_deck/patterns.rb
 - lib/tape_deck/recorder.rb
+- lib/tape_deck/rows.rb
 - lib/tape_deck/session.rb
 - lib/tape_deck/tape.rb
 - lib/tape_deck/version.rb
@@ -71,7 +78,7 @@ licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/fnordfish/tape_deck
-  documentation_uri: https://fnordfish.github.io/tape_deck/docs/v0.1.0.beta1/
+  documentation_uri: https://fnordfish.github.io/tape_deck/docs/v0.1.0/
   changelog_uri: https://github.com/fnordfish/tape_deck/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
 rdoc_options: []