wt_activerecord_index_spy 0.3.0

data/Rakefile

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]
+
+Rake::Task["release:rubygem_push"].clear
+desc "Pick up the .gem file from pkg/ and push it to Gemfury"
+task "release:rubygem_push" do
+  # IMPORTANT: You need to have the `fury` gem installed, and you need to be logged in.
+  # Please DO READ about "impersonation", which is how you push to your company account instead
+  # of your personal account!
+  # https://gemfury.com/help/collaboration#impersonation
+  paths = Dir.glob("#{__dir__}/pkg/*.gem")
+  raise "Must have found only 1 .gem path, but found #{paths.inspect}" if paths.length != 1
+
+  escaped_gem_path = Shellwords.escape(paths.shift)
+  `fury push #{escaped_gem_path} --as=wetransfer`
+end
+
+namespace :db do
+  require_relative "./spec/support/test_database"
+  require "active_record"
+  require "dotenv/load"
+  Dotenv.load
+
+  desc "Create databases to be used in tests"
+  task "create" do
+    adapter = ENV.fetch("ADAPTER", "mysql2")
+    puts "Creating #{adapter}"
+    TestDatabase.set_env_database_url(adapter)
+    TestDatabase.establish_connection
+    ActiveRecord::Base.connection.create_database(TestDatabase.database_name)
+  end
+
+  desc "Drop databases to be used in tests"
+  task "drop" do
+    adapter = ENV.fetch("ADAPTER", "mysql2")
+    puts "Dropping #{adapter}"
+    TestDatabase.set_env_database_url(adapter)
+    TestDatabase.establish_connection
+    ActiveRecord::Base.connection.drop_database(TestDatabase.database_name)
+  end
+
+  desc "Migrate databases to be used in tests"
+  task "migrate" do
+    adapter = ENV.fetch("ADAPTER", "mysql2")
+    puts "Migrating #{adapter}"
+    TestDatabase.set_env_database_url(adapter, with_database_name: true)
+    TestDatabase.establish_connection
+    TestDatabase.run_migrations
+  end
+end

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "wt_activerecord_index_spy"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/wt_activerecord_index_spy.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require_relative "wt_activerecord_index_spy/version"
+require_relative "wt_activerecord_index_spy/aggregator"
+require_relative "wt_activerecord_index_spy/query_analyser"
+require_relative "wt_activerecord_index_spy/query_analyser/mysql"
+require_relative "wt_activerecord_index_spy/query_analyser/postgres"
+require_relative "wt_activerecord_index_spy/notification_listener"
+require "logger"
+
+# This is the top level module which requires everything
+module WtActiverecordIndexSpy
+  extend self
+
+  attr_accessor :logger
+
+  def aggregator
+    @aggregator ||= Aggregator.new
+  end
+
+  def query_analyser
+    @query_analyser ||= QueryAnalyser.new
+  end
+
+  # rubocop:disable Metrics/MethodLength
+  def watch_queries(
+    aggregator: self.aggregator,
+    ignore_queries_originated_in_test_code: true,
+    query_analyser: self.query_analyser
+  )
+    aggregator.reset
+
+    notification_listener = NotificationListener.new(
+      aggregator: aggregator,
+      ignore_queries_originated_in_test_code: ignore_queries_originated_in_test_code,
+      query_analyser: query_analyser
+    )
+
+    subscriber = ActiveSupport::Notifications
+                 .subscribe("sql.active_record", notification_listener)
+
+    return unless block_given?
+
+    yield
+
+    ActiveSupport::Notifications.unsubscribe(subscriber)
+  end
+  # rubocop:enable Metrics/MethodLength
+
+  def export_html_results(file = nil, stdout: $stdout)
+    aggregator.export_html_results(file, stdout: stdout)
+  end
+
+  def certain_results
+    aggregator.certain_results
+  end
+
+  def results
+    aggregator.results
+  end
+
+  def reset_results
+    aggregator.reset
+  end
+
+  def boot
+    @logger = Logger.new("/dev/null")
+  end
+end
+
+WtActiverecordIndexSpy.boot

data/lib/wt_activerecord_index_spy/aggregator.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "erb"
+require "tmpdir"
+
+module WtActiverecordIndexSpy
+  # This class aggregates all queries that were considered not using index.
+  # Since it's not possible to be sure for every query, it separates the result
+  # in certains and uncertains.
+  class Aggregator
+    attr_reader :results
+
+    Item = Struct.new(:identifier, :query, :origin, :certainity_level, keyword_init: true)
+
+    def initialize
+      @results = {}
+    end
+
+    def reset
+      @results = {}
+    end
+
+    # item: an instance of Aggregator::Item
+    def add(item)
+      @results[item.query] = item
+    end
+
+    def certain_results
+      @results.map do |_query, item|
+        item if item.certainity_level == :certain
+      end.compact
+    end
+
+    def uncertain_results
+      @results.map do |_query, item|
+        item if item.certainity_level == :uncertain
+      end.compact
+    end
+
+    def export_html_results(file, stdout: $stdout)
+      file ||= default_html_output_file
+      content = ERB.new(File.read(File.join(File.dirname(__FILE__), "./results.html.erb")), 0, "-")
+                   .result_with_hash(certain_results: certain_results, uncertain_results: uncertain_results)
+
+      file.write(content)
+      file.close
+      stdout.puts "Report exported to #{file.path}"
+    end
+
+    private
+
+    def default_html_output_file
+      File.new(
+        File.join(Dir.tmpdir, "wt_activerecord_index_spy-results.html"),
+        "w"
+      )
+    end
+  end
+end

data/lib/wt_activerecord_index_spy/notification_listener.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module WtActiverecordIndexSpy
+  MissingIndex = Class.new(StandardError)
+
+  # This class can be used to subscribe to an activerecord "sql.active_record"
+  # notification.
+  # It gets each query that uses a WHERE statement and runs a EXPLAIN query to
+  # see if it uses an index.
+  class NotificationListener
+    IGNORED_SQL = [
+      /^PRAGMA (?!(table_info))/,
+      /^SELECT currval/,
+      /^SELECT CAST/,
+      /^SELECT @@IDENTITY/,
+      /^SELECT @@ROWCOUNT/,
+      /^SAVEPOINT/,
+      /^ROLLBACK TO SAVEPOINT/,
+      /^RELEASE SAVEPOINT/,
+      /^SHOW max_identifier_length/,
+      /^SELECT @@FOREIGN_KEY_CHECKS/,
+      /^SET FOREIGN_KEY_CHECKS/,
+      /^TRUNCATE TABLE/,
+      /^EXPLAIN/
+    ].freeze
+
+    attr_reader :queries_missing_index
+
+    def initialize(ignore_queries_originated_in_test_code:,
+                   aggregator: Aggregator.new,
+                   query_analyser: QueryAnalyser.new)
+      @queries_missing_index = []
+      @aggregator = aggregator
+      @query_analyser = query_analyser
+      @ignore_queries_originated_in_test_code = ignore_queries_originated_in_test_code
+    end
+
+    # TODO: refactor me pls to remove all these Rubocop warnings!
+    # rubocop:disable Metrics/AbcSize
+    # rubocop:disable Metrics/MethodLength
+    def call(_name, _start, _finish, _message_id, values)
+      query = values[:sql]
+      logger.debug "query: #{query}"
+      identifier = values[:name]
+
+      if ignore_query?(query: query, name: identifier)
+        logger.debug "query type ignored"
+        return
+      end
+      logger.debug "query type accepted"
+
+      origin = caller.find { |line| !line.include?("/gems/") }
+      if @ignore_queries_originated_in_test_code && query_originated_in_tests?(origin)
+        logger.debug "origin ignored: #{origin}"
+        # Hopefully, it will get the line which executed the query.
+        # It ignores activerecord, activesupport and other gem frames.
+        # Maybe there is a better way to achieve it
+        return
+      end
+
+      logger.debug "origin accepted: #{origin}"
+
+      certainity_level = @query_analyser.analyse(**values.slice(:sql, :connection, :binds))
+      return unless certainity_level
+
+      item = Aggregator::Item.new(
+        identifier: identifier,
+        query: query,
+        origin: reduce_origin(origin),
+        certainity_level: certainity_level
+      )
+
+      @aggregator.add(item)
+    end
+    # rubocop:enable Metrics/AbcSize
+    # rubocop:enable Metrics/MethodLength
+
+    private
+
+    # TODO: Find a better way to detect if the origin is a test file
+    def query_originated_in_tests?(origin)
+      origin.include?("spec/") ||
+        origin.include?("test/")
+    end
+
+    def ignore_query?(name:, query:)
+      # FIXME: this seems bad. we should probably have a better way to indicate
+      # the query was cached
+      name == "CACHE" ||
+        name == "SCHEMA" ||
+        !name ||
+        !query.downcase.include?("where") ||
+        IGNORED_SQL.any? { |r| query =~ r }
+    end
+
+    def reduce_origin(origin)
+      origin[0...origin.rindex(":")]
+        .split("/")[-2..-1]
+        .join("/")
+    end
+
+    def logger
+      WtActiverecordIndexSpy.logger
+    end
+  end
+end

data/lib/wt_activerecord_index_spy/query_analyser.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module WtActiverecordIndexSpy
+  # It runs an EXPLAIN query given a query and analyses the result to see if
+  # some index is missing.
+  class QueryAnalyser
+    def initialize
+      # This is a cache to not run the same EXPLAIN again
+      # It sets the query as key and the result (certain, uncertain) as the value
+      @analysed_queries = {}
+    end
+
+    # The sql and binds vary depend on the adapter.
+    # - Mysql2: sends sql complete and binds = []
+    # - Postregs: sends sql in a form of prepared statement and its values in binds
+    # rubocop:disable Metrics/MethodLength
+    def analyse(sql:, connection: ActiveRecord::Base.connection, binds: [])
+      query = sql
+      # TODO: this could be more intelligent to not duplicate similar queries
+      # with different WHERE values, example:
+      # - WHERE lala = 1 AND popo = 1
+      # - WHERE lala = 2 AND popo = 2
+      # Notes:
+      # - The Postgres adapter uses prepared statements as default, so it
+      # will save the queries without the values.
+      # - The Mysql2 adapter does not use prepared statements as default, so it
+      # will analyse very similar queries as described above.
+      return @analysed_queries[query] if @analysed_queries.key?(query)
+
+      adapter = select_adapter(connection)
+
+      # We need a thread to use a different connection that it's used by the
+      # application otherwise, it can change some ActiveRecord internal state
+      # such as number_of_affected_rows that is returned by the method
+      # `update_all`
+      Thread.new do
+        results = ActiveRecord::Base.connection_pool.with_connection do |conn|
+          conn.exec_query("EXPLAIN #{query}", "SQL", binds)
+        end
+
+        adapter.analyse(results).tap do |certainity_level|
+          @analysed_queries[query] = certainity_level
+        end
+      end.join.value
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    private
+
+    def select_adapter(connection)
+      case connection.adapter_name
+      when "Mysql2"
+        QueryAnalyser::Mysql
+      when "PostgreSQL"
+        QueryAnalyser::Postgres
+      else
+        raise NotImplementedError, "adapter: #{ActiveRecord::Base.connection.adapter_name}"
+      end
+    end
+  end
+end

data/lib/wt_activerecord_index_spy/query_analyser/mysql.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module WtActiverecordIndexSpy
+  class QueryAnalyser
+    # It analyses the result of an EXPLAIN query to see if any index is missing.
+    module Mysql
+      extend self
+
+      ALLOWED_EXTRA_VALUES = [
+        # https://bugs.mysql.com/bug.php?id=64197
+        "Impossible WHERE noticed after reading const tables",
+        "no matching row"
+      ].freeze
+
+      def analyse(results)
+        results.find do |result|
+          certainity_level = analyse_explain(result)
+
+          break certainity_level if certainity_level
+        end
+      end
+
+      private
+
+      # rubocop: disable Metrics/CyclomaticComplexity
+      # rubocop: disable Metrics/PerceivedComplexity
+      def analyse_explain(result)
+        type = result.fetch("type")
+        possible_keys = result.fetch("possible_keys")
+        key = result.fetch("key")
+        extra = result.fetch("Extra")
+
+        # more details about the result in https://dev.mysql.com/doc/refman/8.0/en/explain-output.html
+        return if type == "ref"
+        return if ALLOWED_EXTRA_VALUES.any? { |value| extra&.include?(value) }
+
+        return :certain if possible_keys.nil?
+        return :uncertain if possible_keys == "PRIMARY" && key.nil? && type == "ALL"
+      end
+      # rubocop: enable Metrics/CyclomaticComplexity
+      # rubocop: enable Metrics/PerceivedComplexity
+    end
+  end
+end

data/lib/wt_activerecord_index_spy/query_analyser/postgres.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module WtActiverecordIndexSpy
+  class QueryAnalyser
+    # It analyses the result of an EXPLAIN query to see if any index is missing.
+    module Postgres
+      extend self
+
+      def analyse(results)
+        WtActiverecordIndexSpy.logger.debug("results:\n#{results.rows.join("\n")}")
+
+        full_results = results.rows.join(", ").downcase
+
+        # rubocop:disable Layout/LineLength
+        # Postgres sometimes uses a "seq scan" even for queries that could use an index.
+        # So it's almost impossible to be certain if an index is missing!
+        # The result of the EXPLAIN query varies depending on the state of the database
+        # because Postgres collects statistics from tables and decide if it's better
+        # using an index or not based on that.
+        # This is an example in a real application:
+        #
+        # [1] pry(main)> Feature.where(plan_id: 312312).explain
+        #   Feature Load (4.0ms)  SELECT "features".* FROM "features" WHERE "features"."plan_id" = $1  [["plan_id", 312312]]
+        # => EXPLAIN for: SELECT "features".* FROM "features" WHERE "features"."plan_id" = $1 [["plan_id", 312312]]
+        #                        QUERY PLAN
+        # ---------------------------------------------------------
+        #  Seq Scan on features  (cost=0.00..1.06 rows=1 width=72)
+        #    Filter: (plan_id = 312312)
+        # (2 rows)
+        #
+        # [2] pry(main)> Feature.count
+        #    (2.8ms)  SELECT COUNT(*) FROM "features"
+        # => 5
+        # [3] pry(main)> Plan.count
+        #    (2.7ms)  SELECT COUNT(*) FROM "plans"
+        # => 2
+        #
+        ####################################################################################################################
+        #
+        # [1] pry(main)> Feature.where(plan_id: 312312).explain
+        #   Feature Load (2.3ms)  SELECT "features".* FROM "features" WHERE "features"."plan_id" = $1  [["plan_id", 312312]]
+        # => EXPLAIN for: SELECT "features".* FROM "features" WHERE "features"."plan_id" = $1 [["plan_id", 312312]]
+        #                                        QUERY PLAN
+        # ----------------------------------------------------------------------------------------
+        #  Bitmap Heap Scan on features  (cost=4.18..12.64 rows=4 width=72)
+        #    Recheck Cond: (plan_id = 312312)
+        #    ->  Bitmap Index Scan on index_features_on_plan_id  (cost=0.00..4.18 rows=4 width=0)
+        #          Index Cond: (plan_id = 312312)
+        # rubocop:enable Layout/LineLength
+        return :uncertain if full_results.include?("seq scan on")
+      end
+    end
+  end
+end