columns_trace 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cd2b5b47e4fa2ca409bf7dd531c8766a6ff6487b50e996f04425577c07a65667
+  data.tar.gz: 28fa32928a82d9c1df876bad8002d677bd7c7e5f0a001f56ad6170b9b87e8f2c
+SHA512:
+  metadata.gz: df2740562388acd113084e28f50940d2bbf0d0d934abe0b8b4def515158f2bb36ba29179ae317e7cf77665039e5d77d2c15d76ef044b3e4f56f314f046882ba0
+  data.tar.gz: 77fcfa0230679ba9efc329225c6eda32f6ad915bc0bee79df227b724bc1c75ba3cb5e7b51943b1142ba36586f74d94cdeec9bc9873daa807f1e009c4c53f8e4f

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## master (unreleased)
+
+- First release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 fatkodima
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,119 @@
+# ColumnsTrace
+
+[![Build Status](https://github.com/fatkodima/columns_trace/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/fatkodima/columns_trace/actions/workflows/test.yml)
+
+Detects unnecessary selected database columns in Rails controllers, `ActiveJob` and `Sidekiq` jobs.
+
+## Requirements
+
+- ruby 2.7+
+- rails 6.0+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'columns_trace'
+```
+
+And then run:
+
+```sh
+$ bundle install
+```
+
+## Usage
+
+Hit a controller action or run `ActiveJob` (or `Sidekiq`) job, open `log/columns_trace.log`,
+and see the output:
+
+```
+ImportsController#create
+  1 User record: unused columns - "bio", "settings"; used columns - "id", "email", "name",
+  "account_id", "created_at", "updated_at"
+  ↳ app/controllers/application_controller.rb:32:in `block in <class:ApplicationController>'
+
+  1 Account record: unused columns - "settings", "logo", "updated_at";
+  used columns - "id", "plan_id"
+  ↳ app/controllers/application_controller.rb:33:in `block in <class:ApplicationController>'
+
+  10 Project records: unused columns - "description", "avatar", "url", "created_at", "updated_at";
+  used columns - "id", "user_id"
+  ↳ app/models/user.rb:46: in `projects'
+    app/services/imports_service.rb:129: in `import_projects'
+    app/controllers/imports_controller.rb:49:in `index'
+
+ImportProjectJob
+  1 User record: unused columns - "email", "name", "bio", "created_at", "updated_at";
+  used columns - "id", "settings"
+  ↳ app/jobs/import_project_job.rb:23:in `perform'
+
+  1 Project record: unused columns - "description", "avatar", "settings", "created_at",
+  "updated_at"; used columns - "id", "user_id", "url"
+  ↳ app/jobs/import_project_job.rb:24:in `perform'
+```
+
+## Configuration
+
+You can override the following default options:
+
+```ruby
+# config/initializers/columns_trace.rb
+
+ColumnsTrace.configure do |config|
+  # Configures models that will be ignored.
+  # Always adds Rails' internal `ActiveRecord::SchemaMigration`
+  # and `ActiveRecord::InternalMetadata` models by default.
+  config.ignored_models = []
+
+  # Configures columns that will be ignored.
+  #
+  # Global setting
+  #   config.ignored_columns = [:updated_at]
+  # Per-model setting
+  #   config.ignored_columns = [:updated_at, { User => :admin }]
+  config.ignored_columns = []
+
+  # The logger to log to.
+  # Defaults to logger, that outputs to `log/columns_trace.log` file
+  # when the gem is used in the Rails app.
+  config.logger = nil
+
+  # Controls the contents of the printed backtrace.
+  # Is set to the default Rails.backtrace_cleaner when the gem is used in the Rails app.
+  config.backtrace_cleaner = ->(backtrace) { backtrace }
+end
+```
+
+`Sidekiq` integration is disabled by default. You need to explicitly enable it:
+
+```ruby
+# config/initializers/columns_trace.rb
+
+ColumnsTrace.enable_sidekiq_tracing!
+```
+
+## Development
+
+After checking out the repo, run `bundle install` to install dependencies. Then, run `rake` to run the linter and tests.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Additional resources
+
+Alternatives:
+
+- [snip_snip](https://github.com/kddnewton/snip_snip) - archived, supports only controllers
+
+Interesting reads:
+
+- [Reasons why SELECT * is bad for SQL performance](https://tanelpoder.com/posts/reasons-why-select-star-is-bad-for-sql-performance/)
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/fatkodima/columns_trace.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/columns_trace/rails_integration.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module ColumnsTrace
+  ActiveSupport.on_load(:active_record) do
+    after_find do
+      backtrace = Kernel.caller
+
+      if ColumnsTrace.backtrace_cleaner
+        backtrace = ColumnsTrace.backtrace_cleaner.call(backtrace)
+      end
+
+      Registry.register(self, backtrace) unless ColumnsTrace.ignored_model?(self.class)
+    end
+  end
+
+  ActiveSupport.on_load(:action_controller) do
+    before_action { Registry.clear }
+    after_action { Reporter.report("#{self.class.name}##{action_name}") }
+  end
+
+  ActiveSupport.on_load(:active_job) do
+    before_perform { Registry.clear }
+    after_perform { Reporter.report(self.class.name) }
+  end
+end

data/lib/columns_trace/railtie.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ColumnsTrace
+  # @private
+  class Railtie < Rails::Railtie
+    initializer "columns_trace.set_configs" do
+      ColumnsTrace.backtrace_cleaner = Rails.backtrace_cleaner
+      ColumnsTrace.logger = ActiveSupport::Logger.new(Rails.root.join("log", "columns_trace.log"))
+    end
+  end
+end

data/lib/columns_trace/registry.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module ColumnsTrace
+  # @private
+  # Note: can use ActiveSupport::IsolatedExecutionState instead of this module for rails 7.0+.
+  module Registry
+    Entry = Struct.new(:model, :record, :backtrace)
+
+    class << self
+      def register(record, backtrace)
+        state << Entry.new(record.class, record, backtrace)
+      end
+
+      def clear
+        state.clear
+      end
+
+      def entries
+        state
+      end
+
+      private
+        def state
+          Thread.current[:columns_trace] ||= []
+        end
+    end
+  end
+end

data/lib/columns_trace/reporter.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module ColumnsTrace
+  # @private
+  class Reporter
+    def self.report(title)
+      new.report(title)
+    end
+
+    def report(title)
+      lines = []
+
+      Registry.entries.group_by(&:model).each do |model, entries|
+        lines.concat(lines_for_entries(model, entries))
+      end
+
+      if lines.any?
+        ColumnsTrace.logger.info("#{title}\n#{lines.join("\n")}")
+      end
+    end
+
+    private
+      def lines_for_entries(model, entries)
+        lines = []
+
+        entries.group_by(&:backtrace).each do |backtrace, grouped_entries|
+          records = grouped_entries.map(&:record)
+
+          accessed = accessed_fields(records)
+          unused = records.first.attributes.keys - accessed
+          unused.reject! { |column| ColumnsTrace.ignored_column?(model, column) }
+
+          if unused.any?
+            records_text = "record".pluralize(records.size)
+            lines << <<-MSG
+  #{records.size} #{model.name} #{records_text}: unused columns - #{format_columns(unused)}; used columns - #{format_columns(accessed)}
+  ↳ #{backtrace.join("\n  ")}
+            MSG
+          end
+        end
+
+        lines
+      end
+
+      def accessed_fields(records)
+        records.map(&:accessed_fields).flatten.uniq
+      end
+
+      def format_columns(columns)
+        columns.map(&:inspect).join(", ")
+      end
+  end
+end

data/lib/columns_trace/sidekiq_integration.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module ColumnsTrace
+  # @private
+  class SidekiqMiddleware
+    def call(worker, _job, _queue)
+      Registry.clear
+      yield
+      Reporter.report(worker.class.name)
+    end
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add(ColumnsTrace::SidekiqMiddleware)
+  end
+end

data/lib/columns_trace/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ColumnsTrace
+  VERSION = "0.1.0"
+end

data/lib/columns_trace.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+require_relative "columns_trace/registry"
+require_relative "columns_trace/rails_integration"
+require_relative "columns_trace/reporter"
+require_relative "columns_trace/version"
+require_relative "columns_trace/railtie" if defined?(Rails)
+
+module ColumnsTrace
+  class << self
+    # @private
+    attr_reader :ignored_models
+
+    # Configures models that will be ignored.
+    #
+    # @example
+    #   ColumnsTrace.ignored_models = [Settings]
+    #
+    # Always adds Rails' internal `ActiveRecord::SchemaMigration`
+    # and `ActiveRecord::InternalMetadata` models by default.
+    #
+    def ignored_models=(models)
+      @ignored_models = Array(models) | [ActiveRecord::SchemaMigration, ActiveRecord::InternalMetadata]
+    end
+
+    # @private
+    def ignored_model?(model)
+      ignored_models.include?(model)
+    end
+
+    # Configures columns that will be ignored.
+    #
+    # @example Global setting
+    #   ColumnsTrace.ignored_columns = [:updated_at]
+    # @example Per-model setting
+    #   ColumnsTrace.ignored_columns = [:updated_at, { User => :admin }]
+    #
+    attr_accessor :ignored_columns
+
+    # @private
+    def ignored_column?(model, column)
+      ignored_columns.any? do |value|
+        if value.is_a?(Hash)
+          columns = value[model] || value[model.name] || value[model.name.to_sym]
+          if columns
+            columns = Array(columns).map(&:to_s)
+            columns.include?(column)
+          end
+        else
+          value.to_s == column
+        end
+      end
+    end
+
+    # Allows to set the logger.
+    # Defaults to logger, that outputs to `log/columns_trace.log` file
+    # when inside a rails application.
+    #
+    attr_accessor :logger
+
+    # @private
+    attr_reader :backtrace_cleaner
+
+    # Allows to set a backtrace_cleaner used to clean backtrace before printing them.
+    # Defaults to `Rails.backtrace_cleaner` when inside a rails application.
+    #
+    def backtrace_cleaner=(cleaner)
+      @backtrace_cleaner =
+        if cleaner.respond_to?(:clean)
+          ->(backtrace) { cleaner.clean(backtrace) }
+        else
+          cleaner
+        end
+    end
+
+    # Enables integration with Sidekiq, which is disabled by default.
+    #
+    def enable_sidekiq_tracing!
+      require_relative "columns_trace/sidekiq_integration"
+      true
+    end
+
+    # A convenient method to configure this gem.
+    #
+    # @example
+    #   ColumnsTrace.configure do |config|
+    #     # ...
+    #   end
+    #
+    def configure
+      yield self
+    end
+  end
+
+  self.ignored_models = []
+  self.ignored_columns = []
+  self.backtrace_cleaner = ->(backtrace) { backtrace }
+end

data/test/dummy/tmp/development_secret.txt

@@ -0,0 +1 @@
+3a86c4ee2d1db41bbbe96c557949abd5be8b29f3577316d831a57bd81fe781ae84b631620ebdc3946c5b47747dc9aed358afbdfbf615608ad8c46996b05ab9a7

data/test/dummy/tmp/local_secret.txt

@@ -0,0 +1 @@
+93cbef26b77c38ff5046394a34240e582de1e6d96386235192ed44afd310938076d2f501309478ae03288fd98db9bd43c37f3c50503c0534cc011ac94fa75be4

metadata

@@ -0,0 +1,72 @@
+--- !ruby/object:Gem::Specification
+name: columns_trace
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- fatkodima
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-10-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.0'
+description:
+email:
+- fatkodima123@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- lib/columns_trace.rb
+- lib/columns_trace/rails_integration.rb
+- lib/columns_trace/railtie.rb
+- lib/columns_trace/registry.rb
+- lib/columns_trace/reporter.rb
+- lib/columns_trace/sidekiq_integration.rb
+- lib/columns_trace/version.rb
+- test/dummy/tmp/development_secret.txt
+- test/dummy/tmp/local_secret.txt
+homepage: https://github.com/fatkodima/columns_trace
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/fatkodima/columns_trace
+  source_code_uri: https://github.com/fatkodima/columns_trace
+  changelog_uri: https://github.com/fatkodima/columns_trace/blob/master/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.10
+signing_key:
+specification_version: 4
+summary: Find unnecessary selected database columns.
+test_files: []