database_recorder 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5d51e080ba82dada7af93fbb37380af72e25fef001ecc3f691080048b951f67f
+  data.tar.gz: 5a902b17cab7f30b5cedae2b1e00e374fdc58b3e992cf3bc08d1be8421f37421
+SHA512:
+  metadata.gz: 936d740c398c162be09d7f9631c64ad3c516a55ab787991104df7e32bd53299ebf1d7b50d0f6b605f7118e495a7c7e1d551f2029085d123451291a6c054547f5
+  data.tar.gz: 260f2d6c7bd1d3b65c67a25db41fce5efbd65dc19f294af7e852003c08e78126d58d0f6679b48299ff2ad0646eb96577f8ffc2383e137278e44f222d13326f70

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2022 Mattia Roccoberton
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,81 @@
+# Database Recorder
+
+Record database queries for testing and development purposes only.
+Support only RSpec at the moment, storing logs data on files or Redis.
+
+Main features:
+- store the history of the queries of a test when it run (for monitoring);
+- eventually check if the current queries match the recorded ones (to prevent regressions);
+- [EXPERIMENTAL] optionally replay the recorded queries replacing the original requests.
+
+See below for more details.
+
+## Install
+
+- Add to your Gemfile: `gem 'database_recorder'` (:development, :test groups recommended)
+- With RSpec:
+  + Add to the `spec_helper.rb` (or rails_helper): `DatabaseRecorder::RSpec.setup`
+  + In RSpec examples: add `:dbr` metadata
+  + To verify the matching with the recorded query use: `dbr: { verify_queries: true }`
+
+```rb
+  it 'returns 3 posts', :dbr do
+    # ...
+  end
+```
+
+## Config
+
+Add to your _spec_helper.rb_:
+
+```rb
+# To print the queries while executing the specs: false | true | :color
+DatabaseRecorder::Config.print_queries = true
+
+# Replay the recordings intercepting the queries
+DatabaseRecorder::Config.replay_recordings = true
+
+# To store the queries: :file | :redis | nil
+DatabaseRecorder::Config.storage = :redis
+# nil to avoid storing the queries
+```
+
+## History of the queries
+
+Using the `print_queries` config option is possible to see the executed queries while running the specs. It can be used to identify easily what is going on in a specific example without having to analyze the log files.
+
+Using the `:file` storage, the history is also recorded to files (default path: **spec/dbr**) in YAML format. This is useful for checking what's happening with more details, it includes the query results and some extra data.
+
+## Test queries' changes
+
+This feature can be used to prevent queries regressions.
+It requires to have previously stored the history of the queries (which could be versioned if using file storage).
+It can be activated using `dbr: { verify_queries: true }` metadata.
+
+To work correctly in requires `prepared_statements: true` option in the **database.yml** config file, in the connection block options (available for both Postgres and MySQL).
+
+## Replay the recorded queries
+
+This feature is not stable (at this stage), so use it carefully and supports only deterministic tests (so it doesn't work with Faker, random data or random order specs) and only Postgres is supported for now.
+It requires to have previously stored the history of the queries.
+Using this feature can improve the test suite performances (especially using redis storage).
+It can be activated using `replay_recordings` config option.
+
+Some workarounds to make it works:
+- Run specs with `bin/rspec --order defined`
+- Set a specific seed for Faker (optionally with an ENV var): `Faker::Config.random = Random.new(42)`
+- Set a specific Ruby seed (optionally with an ENV var): `srand(42)`
+
+## Do you like it? Star it!
+
+If you use this component just star it. A developer is more motivated to improve a project when there is some interest.
+
+Or consider offering me a coffee, it's a small thing but it is greatly appreciated: [about me](https://www.blocknot.es/about-me).
+
+## Contributors
+
+- [Mattia Roccoberton](https://blocknot.es): author
+
+## License
+
+The gem is available as open-source under the terms of the [MIT](MIT-LICENSE).

data/lib/database_recorder/activerecord/abstract_adapter_ext.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module DatabaseRecorder
+  module ActiveRecord
+    module AbstractAdapterExt
+      def log(sql, name = 'SQL', binds = [], type_casted_binds = [], *args)
+        # puts "--- #{sql} | #{type_casted_binds}", "  > #{name}"
+        return super unless Recording.started?
+        return super if %w[schema transaction].include?(name&.downcase)
+        return super if sql.downcase.match(/\A(begin|commit|release|rollback|savepoint)/i)
+
+        Core.log_query(sql, name)
+        if Config.replay_recordings && Recording.from_cache
+          Recording.push(sql: sql, binds: binds)
+          data = Recording.cached_query_for(sql)
+          return yield if !data || !data['result'] # cache miss
+
+          RecordedResult.new(data['result']['fields'], data['result']['values'])
+        else
+          super.tap do |result|
+            result_data =
+              if result.is_a?(::ActiveRecord::Result)
+                fields = result.respond_to?(:fields) ? result.fields : result.columns
+                values = result.respond_to?(:values) ? result.values : result.to_a
+                { 'count' => result.count, 'fields' => fields, 'values' => values }
+              end
+            Recording.push(sql: sql, name: name, binds: type_casted_binds, result: result_data)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/database_recorder/activerecord/base_ext.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module DatabaseRecorder
+  module ActiveRecord
+    module BaseExt
+      def create_or_update(**, &block)
+        return super if !new_record? || !Recording.started?
+
+        cached = Config.replay_recordings && Recording.from_cache ? Recording.pull_entity : false
+        if cached
+          # self.id = cached['id']
+          super
+        else
+          super.tap do |_result|
+            Recording.new_entity(model: self.class.to_s, id: id)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/database_recorder/activerecord/recorded_result.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# require 'forwardable'
+
+module DatabaseRecorder
+  module ActiveRecord
+    class RecordedResult < ::ActiveRecord::Result
+      alias :cmd_tuples :count
+      alias :fields :columns
+      alias :values :rows
+
+      def clear; end
+
+      def fmod(_index)
+        -1
+      end
+
+      def ftype(_index)
+        20
+      end
+    end
+  end
+end

data/lib/database_recorder/activerecord/recorder.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module DatabaseRecorder
+  module ActiveRecord
+    module Recorder
+      module_function
+
+      def setup
+        ::ActiveRecord::ConnectionAdapters::AbstractAdapter.class_eval do
+          prepend AbstractAdapterExt
+        end
+
+        # ::ActiveRecord::Base.class_eval do
+        #   prepend BaseExt
+        # end
+      end
+    end
+  end
+end

data/lib/database_recorder/config.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require 'singleton'
+
+module DatabaseRecorder
+  class Config
+    include Singleton
+
+    attr_accessor :db_driver, :print_queries, :replay_recordings, :storage
+
+    class << self
+      extend Forwardable
+
+      def_delegators :instance, :db_driver, :db_driver=, :print_queries, :print_queries=, :replay_recordings,
+                     :replay_recordings=, :storage
+
+      def load_defaults
+        instance.db_driver = :active_record
+        instance.print_queries = false # false | true | :color
+        instance.replay_recordings = false
+        self.storage = :file # :file | :redis
+      end
+
+      def storage=(value)
+        instance.storage =
+          case value
+          when :file then DatabaseRecorder::Storage::File
+          when :redis then DatabaseRecorder::Storage::Redis
+          when nil then nil
+          else raise ArgumentError, "Unknown storage: #{value}"
+          end
+      end
+    end
+  end
+end
+
+DatabaseRecorder::Config.load_defaults

data/lib/database_recorder/core.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module DatabaseRecorder
+  module Core
+    module_function
+
+    def log_query(sql, source = nil)
+      case DatabaseRecorder::Config.print_queries
+      when true
+        puts "[DB] #{sql} [#{source}]"
+      when :color
+        puts "[DB] #{CodeRay.scan(sql, :sql).term} [#{source}]"
+      end
+    end
+
+    def setup
+      case DatabaseRecorder::Config.db_driver
+      when :active_record then ActiveRecord::Recorder.setup
+      when :mysql2 then Mysql2.setup
+      when :pg then PG.setup
+      end
+    end
+  end
+end

data/lib/database_recorder/mysql2/recorded_result.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# require 'forwardable'
+
+module DatabaseRecorder
+  module Mysql2
+    class RecordedResult # < ::Mysql2::Result
+      # include Enumerable
+      # extend Forwardable
+
+      # def_delegators :to_a, :each
+
+      attr_reader :count, :entries, :fields # , :values
+
+      alias :size :count
+
+      def prepare(data)
+        @count = data['count']
+        @fields = data['fields']
+        @entries = data['values']
+        # @values = data['values']
+      end
+
+      # def server_flags
+      #   { no_good_index_used: false, no_index_used: true, query_was_slow: false }
+      # end
+
+      # def to_a
+      #   @to_a ||= values.each_with_object([]) do |value, list|
+      #     list << fields.zip(value).to_h
+      #   end
+      # end
+    end
+  end
+end

data/lib/database_recorder/mysql2/recorder.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module DatabaseRecorder
+  module Mysql2
+    module Recorder
+      def query(sql, options = {})
+        Mysql2.record(self, sql: sql) do
+          super
+        end
+      end
+    end
+
+    module_function
+
+    def ignore_query?(sql)
+      !Recording.started? ||
+        sql == 'SELECT LAST_INSERT_ID() AS _dbr_last_insert_id' ||
+        sql.downcase.match?(/\A(begin|commit|release|rollback|savepoint|show full fields from)/i) ||
+        sql.match?(/information_schema.statistics/)
+    end
+
+    def record(recorder, sql:)
+      return yield if ignore_query?(sql)
+
+      Core.log_query(sql)
+      if Config.replay_recordings && !Recording.cache.nil?
+        Recording.push(sql: sql)
+        data = Recording.cached_query_for(sql)
+        return yield unless data # cache miss
+
+        RecordedResult.new.prepare(data['result'].slice('count', 'fields', 'values')) if data['result']
+      else
+        yield.tap do |result|
+          result_data =
+            if result
+              { 'count' => result.count, 'fields' => result.fields, 'values' => result.to_a }
+            else
+              last_insert_id = recorder.query('SELECT LAST_INSERT_ID() AS _dbr_last_insert_id').to_a
+              { 'count' => last_insert_id.count, 'fields' => ['id'], 'values' => last_insert_id }
+            end
+
+          Recording.push(sql: sql, result: result_data)
+        end
+      end
+    end
+
+    def setup
+      ::Mysql2::Client.class_eval do
+        prepend Recorder
+      end
+    end
+  end
+end

data/lib/database_recorder/pg/recorded_result.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module DatabaseRecorder
+  module PG
+    class RecordedResult
+      attr_reader :count, :fields, :values, :type_map
+
+      alias :cmd_tuples :count
+      alias :columns :fields
+      alias :ntuples :count
+      alias :rows :values
+
+      def initialize(data)
+        @count = data['count']
+        @fields = data['fields']
+        @values = data['values']
+      end
+
+      def clear; end
+
+      def first
+        to_a.first
+      end
+
+      def fmod(_index)
+        -1
+      end
+
+      def ftype(_index)
+        20
+      end
+
+      def map_types!(type_map)
+        @type_map = type_map
+        self
+      end
+
+      def nfields
+        fields.size
+      end
+
+      def to_a
+        @to_a ||= values.each_with_object([]) do |value, list|
+          list << fields.zip(value).to_h
+        end
+      end
+    end
+  end
+end

data/lib/database_recorder/pg/recorder.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module DatabaseRecorder
+  module PG
+    module Recorder
+      def async_exec(sql)
+        PG.record(sql: sql, source: :async_exec) do
+          super
+        end
+      end
+
+      def sync_exec(sql)
+        PG.record(sql: sql, source: :sync_exec) do
+          super
+        end
+      end
+
+      # def exec(*args)
+      #   puts "-E- #{args[0]}"
+      #   super
+      # end
+
+      # def query(*args)
+      #   puts "-Q- #{args[0]}"
+      #   super
+      # end
+
+      def exec_params(*args)
+        PG.record(sql: args[0], binds: args[1], source: :exec_params) do
+          super
+        end
+      end
+
+      # def async_exec_params(*args)
+      #   puts ">>> #{args[0]}"
+      #   super
+      # end
+
+      # def sync_exec_params(*args)
+      #   puts ">>> #{args[0]}"
+      #   super
+      # end
+
+      def exec_prepared(*args)
+        PG.record(sql: args[0], binds: args[1], source: :exec_prepared) do
+          super
+        end
+      end
+    end
+
+    module_function
+
+    def ignore_query?(sql)
+      !Recording.started? ||
+        sql.downcase.match?(/\A(begin|commit|release|rollback|savepoint)/i) ||
+        sql.match?(/ pg_attribute |SHOW max_identifier_length|SHOW search_path/)
+    end
+
+    def record(sql:, binds: nil, source: nil)
+      return yield if ignore_query?(sql)
+
+      Core.log_query(sql, source)
+      if Config.replay_recordings && !Recording.cache.nil?
+        Recording.push(sql: sql, binds: binds)
+        data = Recording.cached_query_for(sql)
+        return yield unless data # cache miss
+
+        RecordedResult.new(data['result'].slice('count', 'fields', 'values'))
+      else
+        yield.tap do |result|
+          result_data = result ? { 'count' => result.count, 'fields' => result.fields, 'values' => result.values } : nil
+          Recording.push(sql: sql, binds: binds, result: result_data)
+        end
+      end
+    end
+
+    def setup
+      ::PG::Connection.class_eval do
+        prepend Recorder
+      end
+    end
+  end
+end

data/lib/database_recorder/recording.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module DatabaseRecorder
+  class Recording
+    attr_accessor :cache, :entities
+    attr_reader :from_cache, :options, :queries, :started
+
+    def initialize(options: {})
+      (@@instances ||= {})[Process.pid] = self
+      @cache = nil
+      @entities = []
+      @options = options
+      @queries = []
+      @search_index = 0
+    end
+
+    def cached_query_for(sql)
+      current = @search_index
+      match = cache[@search_index..].find do |item|
+        current += 1
+        item['sql'] == sql # TODo: try matching by normalized query (no values)
+      end
+      return unless match
+
+      @search_index = current
+      match
+
+      # cache.shift # TMP
+    end
+
+    def new_entity(model:, id:)
+      @entities.push('model' => model, 'id' => id)
+    end
+
+    def pull_entity
+      @entities.shift
+    end
+
+    def push(sql:, binds: nil, result: nil, name: nil)
+      query = { 'name' => name, 'sql' => sql, 'binds' => binds, 'result' => result }.compact
+      @queries.push(query)
+    end
+
+    def start
+      @started = true
+      storage = Config.storage&.new(self, name: options[:name])
+      @from_cache = storage&.load
+      yield
+      storage&.save unless from_cache
+      @started = false
+      result = { current_queries: queries.map { _1['sql'] } }
+      result[:stored_queries] = cache.map { _1['sql'] } if from_cache
+      result
+    end
+
+    class << self
+      extend Forwardable
+
+      def_delegators :current_instance, :cache, :cached_query_for, :from_cache, :new_entity, :pull_entity, :push
+
+      def current_instance
+        (@@instances ||= {})[Process.pid]
+      end
+
+      def started?
+        current_instance&.started
+      end
+    end
+  end
+end

data/lib/database_recorder/rspec.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module DatabaseRecorder
+  module RSpec
+    module_function
+
+    def setup
+      ::RSpec.configure do |config|
+        config.before(:suite) do
+          Core.setup
+        end
+
+        config.around(:each, :dbr) do |example|
+          ref = (example.metadata[:scoped_id] || '').split(':')[-1]
+          options = {}
+          options.merge!(example.metadata[:dbr]) if example.metadata[:dbr].is_a?(Hash)
+          options.merge!(example: example)
+          options.merge!(name: "#{example.full_description}__#{ref}") # TODO: if name is already set, append ref
+          Recording.new(options: options).tap do |recording|
+            result = recording.start { example.run }
+            if options[:verify_queries] && result[:stored_queries]
+              expect(result[:stored_queries]).to match_array(result[:current_queries])
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/database_recorder/storage/file.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module DatabaseRecorder
+  module Storage
+    class File
+      def initialize(recording, name:)
+        @recording = recording
+        @name = name
+      end
+
+      def load
+        stored_data = ::File.exist?(record_file) ? ::File.read(record_file) : false
+        if stored_data
+          data = YAML.load(stored_data)
+          @recording.cache = data['queries']
+          @recording.entities = data['entities']
+          true
+        else
+          false
+        end
+      end
+
+      def save
+        data = { 'queries' => @recording.queries }
+        data['entities'] = @recording.entities if @recording.entities.any?
+        serialized_data = data.to_yaml
+        ::File.write(record_file, serialized_data)
+      end
+
+      private
+
+      def normalize_name(string)
+        string.gsub(%r{[:/]}, '-').gsub(/[^\w-]/, '_')
+      end
+
+      def record_file
+        name = normalize_name(@name)
+        path = 'spec/dbr'
+        FileUtils.mkdir_p(path)
+        "#{path}/#{name}.yml"
+      end
+    end
+  end
+end

data/lib/database_recorder/storage/redis.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module DatabaseRecorder
+  module Storage
+    class Redis
+      def initialize(recording, name:)
+        @recording = recording
+        @name = name
+      end
+
+      def load
+        stored_data = self.class.connection.get(@name)
+        if stored_data
+          data = JSON.parse(stored_data)
+          @recording.cache = data['queries']
+          @recording.entities = data['entities']
+          true
+        else
+          false
+        end
+      end
+
+      def save
+        data = { 'queries' => @recording.queries }
+        data['entities'] = @recording.entities if @recording.entities.any?
+        serialized_data = data.to_json
+        self.class.connection.set(@name, serialized_data)
+      end
+
+      class << self
+        def connection
+          @connection ||= ::Redis.new
+        end
+      end
+    end
+  end
+end

data/lib/database_recorder/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DatabaseRecorder
+  VERSION = '0.1.0'
+end

data/lib/database_recorder.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require_relative 'database_recorder/core'
+
+if defined? ::ActiveRecord
+  require_relative 'database_recorder/activerecord/abstract_adapter_ext'
+  require_relative 'database_recorder/activerecord/base_ext'
+  require_relative 'database_recorder/activerecord/recorded_result'
+  require_relative 'database_recorder/activerecord/recorder'
+end
+
+if defined? ::Mysql2
+  require_relative 'database_recorder/mysql2/recorded_result'
+  require_relative 'database_recorder/mysql2/recorder'
+end
+
+if defined? ::PG
+  require_relative 'database_recorder/pg/recorded_result'
+  require_relative 'database_recorder/pg/recorder'
+end
+
+require_relative 'database_recorder/storage/file'
+require_relative 'database_recorder/storage/redis'
+
+require_relative 'database_recorder/rspec'
+require_relative 'database_recorder/recording'
+require_relative 'database_recorder/config'

metadata

@@ -0,0 +1,79 @@
+--- !ruby/object:Gem::Specification
+name: database_recorder
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Mattia Roccoberton
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2022-04-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: coderay
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+description: Record application queries, verify them against stored queries, and replay
+  them.
+email:
+- mat@blocknot.es
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- lib/database_recorder.rb
+- lib/database_recorder/activerecord/abstract_adapter_ext.rb
+- lib/database_recorder/activerecord/base_ext.rb
+- lib/database_recorder/activerecord/recorded_result.rb
+- lib/database_recorder/activerecord/recorder.rb
+- lib/database_recorder/config.rb
+- lib/database_recorder/core.rb
+- lib/database_recorder/mysql2/recorded_result.rb
+- lib/database_recorder/mysql2/recorder.rb
+- lib/database_recorder/pg/recorded_result.rb
+- lib/database_recorder/pg/recorder.rb
+- lib/database_recorder/recording.rb
+- lib/database_recorder/rspec.rb
+- lib/database_recorder/storage/file.rb
+- lib/database_recorder/storage/redis.rb
+- lib/database_recorder/version.rb
+homepage: https://github.com/blocknotes/database_recorder
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/blocknotes/database_recorder
+  source_code_uri: https://github.com/blocknotes/database_recorder
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.6
+signing_key:
+specification_version: 4
+summary: A SQL database recorder
+test_files: []