rails_pulse 0.3.3 → 0.4.0.pre.2

data/lib/generators/rails_pulse/templates/rails_pulse.rb

@@ -6,6 +6,11 @@ RailsPulse.configure do |config|
   # Enable or disable Rails Pulse
   config.enabled = true
 
+  # Tracking writes happen on a background thread by default (see `config.async`
+  # under ADVANCED). Transactional tests share one database connection across
+  # threads, so write inline there to keep the writer off the test's connection.
+  config.async = false if Rails.env.test?
+
   # ====================================================================================================
   #                                               THRESHOLDS
   # ====================================================================================================
@@ -116,6 +121,58 @@ RailsPulse.configure do |config|
 
   config.tags = [ "ignored", "critical", "experimental" ]
 
+  # ====================================================================================================
+  #                                          EXCEPTION TRACKING
+  # ====================================================================================================
+  # When enabled, Rails Pulse captures unhandled exceptions raised during web
+  # requests and failed background jobs, groups them by class and location, and
+  # displays them in the Exceptions tab. Rake tasks are not captured automatically —
+  # call ExceptionCaptureService.capture yourself if needed.
+  #
+  # Capture runs synchronously on the calling thread (upsert + insert). That keeps
+  # the implementation simple for v1; under an error storm it adds DB latency to
+  # failing requests and jobs. Set track_exceptions = false to disable.
+  #
+  # The gem default is false so existing installs do not start capturing on
+  # upgrade. New apps get true from this template; the upgrade generator
+  # inserts false into existing initializers.
+  #
+  # Backtraces store the first 50 frames. Request params are filtered via Rails'
+  # filter_parameters. Exception messages are filtered too: the SQL and echoed
+  # values in ActiveRecord::StatementInvalid messages are stripped, and any
+  # `key=value` / `key: value` fragment whose key matches filter_parameters
+  # (password=…, token: …) is masked. Messages are truncated to 500 characters.
+
+  # Enable or disable exception tracking
+  config.track_exceptions = true
+
+  # Capture request params with each exception occurrence.
+  # Params are filtered using Rails' filter_parameters config (passwords, tokens, etc. are redacted).
+  # Occurrences with params larger than 10KB after filtering are stored without params.
+  # Set to false to disable entirely, e.g. for strict data-minimization requirements.
+  config.capture_exception_params = true
+
+  # Extra redaction for exception messages, applied after the built-in
+  # filter_parameters pass. Receives the message and the exception; return the
+  # message to store. If the hook raises, the message is stored as "[FILTERED]".
+  # config.exception_message_filter = ->(message, exception) {
+  #   message.gsub(/\b\d{13,16}\b/, "[FILTERED]")   # card-number-shaped digits
+  # }
+
+  # How many times an exception group has to fire over the dashboard period
+  # before it is called out. These are occurrence counts, not durations.
+  # config.exception_thresholds = {
+  #   warning:  10,
+  #   critical: 100
+  # }
+
+  # Capture the raw (unparameterized) SQL for each operation.
+  # WARNING: mysql2 defaults to prepared_statements: false, so every literal
+  # value (emails, passwords, tokens) is inlined and stored in plaintext.
+  # Same applies to PostgreSQL behind PgBouncer in transaction-pool mode.
+  # Default: false (upgrade-safe). New installs may opt in after review.
+  # config.capture_actual_sql = true
+
   # ====================================================================================================
   #                                            BACKGROUND JOBS
   # ====================================================================================================
@@ -144,8 +201,21 @@ RailsPulse.configure do |config|
   config.ignored_queues = []
 
   # Capture job arguments for debugging (may contain sensitive data)
-  # Set to false in production to avoid storing potentially sensitive information
-  config.capture_job_arguments = true
+  # WARNING: job arguments may contain user credentials, PII, or API keys.
+  # Default: false. Set to true only after reviewing your job argument contents.
+  config.capture_job_arguments = false
+
+  # Job tracking mode: :universal (all jobs) or :opt_in (only explicitly tracked jobs)
+  # config.job_tracking_mode = :universal
+
+  # Per-adapter settings. Disable adapters or opt into queue-depth tracking:
+  # config.job_adapters = {
+  #   sidekiq: { enabled: true, track_queue_depth: false },
+  #   solid_queue: { enabled: true, track_recurring: false },
+  #   good_job: { enabled: true, track_cron: false },
+  #   delayed_job: { enabled: true },
+  #   resque: { enabled: true }
+  # }
 
   # ====================================================================================================
   #                                            DATABASE CONFIGURATION
@@ -177,26 +247,37 @@ RailsPulse.configure do |config|
   #     password: <%= Rails.application.credentials.dig(:rails_pulse, :database_password) %>
   #     host: localhost
   #     pool: 5
+  #     migrations_paths: db/rails_pulse_migrate
+  #     schema_dump: false
 
   # ====================================================================================================
   #                                            AUTHENTICATION
   # ====================================================================================================
   # Configure authentication to secure access to the Rails Pulse dashboard.
-  # Authentication is ENABLED BY DEFAULT in production environments for security.
+  # Authentication is ENABLED BY DEFAULT outside development and test.
   #
-  # If no authentication method is configured, Rails Pulse will use HTTP Basic Auth
+  # If nothing below is configured, Rails Pulse will use HTTP Basic Auth
   # with credentials from RAILS_PULSE_USERNAME (default: 'admin') and RAILS_PULSE_PASSWORD
   # environment variables. Set RAILS_PULSE_PASSWORD to enable this fallback.
-  #
-  # Uncomment and configure one of the following patterns based on your authentication system:
 
-  # Enable/disable authentication (enabled by default in production)
-  # config.authentication_enabled = Rails.env.production?
+  # Enable/disable authentication (enabled by default outside development/test)
+  # config.authentication_enabled = !Rails.env.local?
 
-  # Where to redirect unauthorized users
+  # Where to redirect users when an authentication hook raises
   # config.authentication_redirect_path = "/"
 
-  # Custom authentication method - choose one of the examples below:
+  # RECOMMENDED: a fail-closed predicate. Receives the controller; return
+  # true to allow. Anything else (false, nil, no user) is a 403 Forbidden.
+  # config.authorize = ->(controller) { controller.current_user&.admin? }
+  #
+  # A zero-argument proc runs in the controller's context instead:
+  # config.authorize = proc { user_signed_in? && current_user.admin? }
+
+  # Alternatively, a hook that runs in the controller and DENIES BY RENDERING
+  # OR REDIRECTING. Returning false without responding is also a denial;
+  # returning nil (what `unless … end` returns on success) allows the request.
+  # Do not write predicate-style checks here — use config.authorize for those.
+  # If both are set, authentication_method runs first, then authorize.
 
   # Example 1: Devise with admin role check
   # config.authentication_method = proc {
@@ -220,7 +301,8 @@ RailsPulse.configure do |config|
   # Example 4: Basic HTTP authentication
   # config.authentication_method = proc {
   #   authenticate_or_request_with_http_basic do |username, password|
-  #     username == ENV['RAILS_PULSE_USERNAME'] && password == ENV['RAILS_PULSE_PASSWORD']
+  #     ActiveSupport::SecurityUtils.secure_compare(username, ENV['RAILS_PULSE_USERNAME']) &&
+  #       ActiveSupport::SecurityUtils.secure_compare(password, ENV['RAILS_PULSE_PASSWORD'])
   #   end
   # }
 
@@ -246,6 +328,10 @@ RailsPulse.configure do |config|
   #   config.deployment_api_token = Rails.application.credentials.dig(:rails_pulse, :deployment_api_token)
   #   config.deployment_api_token = ENV["RAILS_PULSE_DEPLOYMENT_TOKEN"]
   #
+  # Limits: revision ≤ 255 characters, metadata ≤ 4 KB serialized, started_at at most
+  # one hour in the future. Rows beyond max_table_records[:rails_pulse_deployments]
+  # are pruned oldest-first by the cleanup task.
+  #
   # Record a deployment from your CI/CD pipeline:
   #   curl -X POST https://yourapp.com/rails_pulse/deployments \
   #     -H "X-Rails-Pulse-Token: $RAILS_PULSE_DEPLOYMENT_TOKEN" \
@@ -273,15 +359,70 @@ RailsPulse.configure do |config|
   config.archiving_enabled = true
 
   # Time-based retention - delete records older than this period
-  config.full_retention_period = 2.weeks
+  config.full_retention_period = 30.days
 
   # Count-based retention - maximum records to keep per table
   # After time-based cleanup, if tables still exceed these limits,
   # the oldest remaining records will be deleted to stay under the limit
   config.max_table_records = {
-    rails_pulse_requests: 10000,    # HTTP requests (moderate volume)
-    rails_pulse_operations: 50000,  # Operations within requests (high volume)
-    rails_pulse_routes: 1000,       # Unique routes (low volume)
-    rails_pulse_queries: 500        # Normalized SQL queries (low volume)
+    rails_pulse_requests: 50_000,                 # HTTP requests (moderate volume)
+    rails_pulse_operations: 100_000,              # Operations within requests (high volume)
+    rails_pulse_routes: 1_000,                    # Unique routes (low volume)
+    rails_pulse_queries: 10_000,                  # Normalized SQL queries (low volume)
+    rails_pulse_job_runs: 50_000,                 # Individual job executions (high volume)
+    rails_pulse_jobs: 1_000,                      # Unique job classes (low volume)
+    rails_pulse_exception_occurrences: 50_000,    # Individual exception raises (high volume)
+    rails_pulse_exception_groups: 10_000,         # Distinct exception sites (moderate volume)
+    rails_pulse_deployments: 1_000                # Deploy markers (low volume; oldest pruned first)
   }
+
+  # ====================================================================================================
+  #                                       HISTORICAL COMPARISON
+  # ====================================================================================================
+
+  # Rails Pulse compares recent behaviour against a route, query or job's own
+  # history to answer "what changed?" rather than only "what is slow?".
+  #
+  # The baseline is the traffic-weighted metric across `baseline_window` of day
+  # summaries; `comparison_window` is the recent slice measured against it. The
+  # two never overlap.
+  # config.baseline_window   = 28.days
+  # config.comparison_window = 1.day
+
+  # Hourly summaries decide how precisely a change point can be placed. They are
+  # pruned at this age because the 1-day view is otherwise the only thing that
+  # reads them. Inside this window Rails Pulse can say a route slowed down at
+  # 14:00; beyond it, the finest answer is the day. Raising this buys precision
+  # at the cost of summary table growth.
+  # config.hourly_summary_retention = 2.days
+
+  # A change is reported as a regression only when it clears both the ratio and
+  # the absolute floor for its unit. The ratio alone flags trivial millisecond
+  # noise on fast endpoints; the floor alone flags slow endpoints that never
+  # actually changed. min_samples keeps low-traffic subjects quiet.
+  # config.regression_thresholds = {
+  #   ratio:                1.5,   # 50% worse than baseline
+  #   min_delta_ms:         50.0,  # ...and at least 50ms worse
+  #   min_delta_rate:       1.0,   # ...or 1 percentage point, for error rates
+  #   min_samples:          100,   # minimum observations on each side
+  #   min_baseline_periods: 3      # minimum days of history before comparing
+  # }
+
+  # ====================================================================================================
+  #                                               ADVANCED
+  # ====================================================================================================
+
+  # Use a custom logger (default: Rails.logger)
+  # config.logger = Logger.new("log/rails_pulse.log")
+
+  # Perform tracking writes in a background thread (default: true).
+  # When false, writes happen inline before the response is sent.
+  # config.async = true
+
+  # Show a dashboard banner when summary data is stale (default: true)
+  # config.warn_on_stale_summaries = true
+
+  # Set to false to skip dashboard middleware and asset serving entirely.
+  # Useful for standalone/API-only deployments that use a separate dashboard app.
+  # config.mount_dashboard = true
 end

data/lib/generators/rails_pulse/upgrade_generator.rb

@@ -1,5 +1,6 @@
 require_relative "base_methods"
 require_relative "schema_parser"
+require_relative "../../rails_pulse/installers/config_updater"
 
 module RailsPulse
   module Generators
@@ -21,9 +22,10 @@ module RailsPulse
 
         case @database_type
         when :single
-          upgrade_installation(migration_dir: "db/migrate", next_steps: single_db_next_steps)
+          upgrade_installation(migration_dir: "db/migrate", migrate_command: "rails db:migrate")
         when :separate
-          upgrade_installation(migration_dir: "db/rails_pulse_migrate", next_steps: separate_db_next_steps)
+          warn_if_missing_schema_dump_false
+          upgrade_installation(migration_dir: "db/rails_pulse_migrate", migrate_command: "rails db:migrate:rails_pulse")
         when :schema_only
           offer_conversion_to_migrations
         when :not_installed
@@ -38,16 +40,20 @@ module RailsPulse
         # Override with command line option if provided
         return options[:database].to_sym if options[:database] != "detect"
 
-        # Check for existing Rails Pulse tables
+        # Determine database type before checking tables so the table lookup
+        # uses the correct connection pool for separate-database setups.
+        @is_separate_db = has_separate_database_config?
         tables_exist = rails_pulse_tables_exist?
-
         schema_path = File.join(root_path, "db/rails_pulse_schema.rb")
 
-        if !tables_exist && File.exist?(schema_path)
+        # :schema_only only applies to single-database users who have the schema file
+        # but haven't run the install migration yet. Separate-database users legitimately
+        # have no Rails Pulse tables on the primary connection — that is not an error.
+        if !tables_exist && !@is_separate_db && File.exist?(schema_path)
           :schema_only
         elsif !tables_exist
           :not_installed
-        elsif has_separate_database_config?
+        elsif @is_separate_db
           :separate
         else
           :single
@@ -56,25 +62,30 @@ module RailsPulse
 
       def has_separate_database_config?
         config_path = File.join(root_path, "config/database.yml")
-
         return false unless File.exist?(config_path)
 
         require "yaml"
-        db_config = YAML.safe_load(File.read(config_path), aliases: true)
-
-        # Check if any environment has a rails_pulse database configuration
+        require "erb"
+        # Process ERB before YAML parsing — database.yml files commonly use ERB
+        # for environment-specific values. YAML.safe_load alone raises SyntaxError
+        # on ERB tags.
+        yaml_content = ERB.new(File.read(config_path)).result
+        db_config = YAML.safe_load(yaml_content, aliases: true)
         db_config.values.any? { |env| env.is_a?(Hash) && env.key?("rails_pulse") }
-      rescue Psych::SyntaxError, Psych::AliasesNotEnabled, Errno::ENOENT
-        # If we can't read or parse the file, assume single database
+      rescue
         false
       end
 
       def rails_pulse_tables_exist?
         return false unless defined?(ActiveRecord::Base)
 
-        connection = ActiveRecord::Base.connection
-        required_tables = get_rails_pulse_table_names
+        connection = if @is_separate_db && defined?(RailsPulse::ApplicationRecord)
+          RailsPulse::ApplicationRecord.connection
+        else
+          ActiveRecord::Base.connection
+        end
 
+        required_tables = get_rails_pulse_table_names
         required_tables.all? { |table| connection.table_exists?(table) }
       rescue
         false
@@ -85,8 +96,41 @@ module RailsPulse
         SchemaParser.new(schema_file).extract_table_names
       end
 
+      # Notices printed when a feature's migration is newly copied into the host app.
+      # Key is a substring of the migration filename (without timestamp).
+      FEATURE_NOTICES = {
+        "create_rails_pulse_exceptions" => <<~NOTICE.rstrip
+          Exception tracking (opt-in)
+
+          This upgrade adds exception tables. Capture stays off for existing
+          installs until you opt in. After migrating, set the following in
+          config/initializers/rails_pulse.rb (the upgrade generator inserts
+          it as false — review with git diff) and restart:
+
+            config.track_exceptions = true
+
+          New installs enable this in the generated initializer. Messages and
+          request params are both redacted using Rails filter_parameters; set
+          config.exception_message_filter for anything app-specific.
+
+          Two new tables will be created:
+            - rails_pulse_exception_groups
+            - rails_pulse_exception_occurrences
+
+          To skip the tables entirely, delete the copied
+          *_create_rails_pulse_exceptions.rb migration before db:migrate.
+          (A later upgrade will copy it again unless that file remains in
+          your migrate folder.)
+        NOTICE
+      }.freeze
+
       # Shared upgrade logic for both single and separate database setups
-      def upgrade_installation(migration_dir:, next_steps:)
+      def upgrade_installation(migration_dir:, migrate_command:)
+        # Refresh the schema file so fresh databases (test, CI) built from
+        # db/rails_pulse_schema.rb include all current columns and tables.
+        copy_file "db/rails_pulse_schema.rb", "db/rails_pulse_schema.rb", force: true
+        sync_initializer
+
         gem_migrations = get_gem_migrations
         existing_migrations = get_user_migrations(migration_dir)
         new_migrations = gem_migrations - existing_migrations
@@ -99,18 +143,38 @@ module RailsPulse
           end
 
           say "\nMigrations copied successfully!", :green
-          say "\nNext steps:", :green
-          next_steps.each { |step| say step }
+          announce_new_features(new_migrations)
+          include_route_backfill = requires_route_backfill?(new_migrations)
+          say_route_backfill_warning if include_route_backfill
+          say_next_steps(migrate_command, include_route_backfill: include_route_backfill)
         else
-          upgrade_with_missing_columns(migration_dir: migration_dir, next_steps: next_steps)
+          upgrade_with_missing_columns(migration_dir: migration_dir, migrate_command: migrate_command)
         end
       end
 
-      def upgrade_with_missing_columns(migration_dir:, next_steps:)
+      def announce_new_features(new_migrations)
+        notices = FEATURE_NOTICES.filter_map do |migration_key, notice|
+          notice if new_migrations.any? { |filename| filename.include?(migration_key) }
+        end
+        return if notices.empty?
+
+        say "\n" + ("=" * 72), :yellow
+        notices.each_with_index do |notice, index|
+          say "" if index.positive?
+          notice.each_line { |line| say line.chomp, :yellow }
+        end
+        say ("=" * 72) + "\n", :yellow
+      end
+
+      def upgrade_with_missing_columns(migration_dir:, migrate_command:)
         missing_columns = detect_missing_columns
 
         if missing_columns.empty?
-          say "Rails Pulse is up to date! No migration needed.", :green
+          if @initializer_updated
+            say "Schema is up to date. Review initializer changes with git diff.", :green
+          else
+            say "Rails Pulse is up to date! No migration needed.", :green
+          end
           return
         end
 
@@ -126,31 +190,91 @@ module RailsPulse
           "#{migration_dir}/upgrade_rails_pulse_tables.rb"
         )
 
-        say <<~MESSAGE
+        say "\nUpgrade migration created successfully!", :green
+        missing_names = missing_columns.keys.map(&:to_s)
+        include_route_backfill = missing_names.intersect?(%w[controller_action http_methods])
+        say_route_backfill_warning if include_route_backfill
+        say_next_steps(migrate_command, include_route_backfill: include_route_backfill)
+        say "\nThis migration will add: #{missing_columns.keys.join(', ')}\n"
+      end
 
-          Upgrade migration created successfully!
+      ROUTE_BACKFILL_MIGRATIONS = %w[
+        change_rails_pulse_routes_to_multi_verb_model
+        add_null_action_unique_index_to_routes
+      ].freeze
 
-          Next steps:
-          #{next_steps.map { |s| "  #{s}" }.join("\n")}
+      def requires_route_backfill?(migrations)
+        migrations.any? do |name|
+          ROUTE_BACKFILL_MIGRATIONS.any? { |fragment| name.include?(fragment) }
+        end
+      end
 
-          This migration will add: #{missing_columns.keys.join(', ')}
+      def say_route_backfill_warning
+        say "\nIMPORTANT: This upgrade changes how routes are identified.", :yellow
+        say "Schema migrate alone leaves the Action column empty. After migrating, run:", :yellow
+        say "  rails rails_pulse:migrate_routes", :yellow
+        say "Skipping this leaves GET/POST on the same path unmerged in the dashboard.", :yellow
+        say "\nWARNING: Migration 20260610000002 is irreversible (drops routes.method).", :red
+        say "db:rollback past this point requires a database restore.", :red
+      end
 
-        MESSAGE
+      def say_next_steps(migrate_command, include_route_backfill:)
+        n = 1
+        say "\nNext steps:", :green
+        say "#{n}. Run: #{migrate_command}"
+        n += 1
+        if include_route_backfill
+          say "#{n}. Run: rails rails_pulse:migrate_routes"
+          n += 1
+        end
+        if @initializer_updated
+          say "#{n}. Review config/initializers/rails_pulse.rb with git diff"
+          n += 1
+        end
+        say "#{n}. Restart ALL processes (web + workers) together — a rolling restart"
+        say "   that leaves old processes against the new schema breaks tracking."
+      end
+
+      def sync_initializer
+        path = File.join(root_path, "config/initializers/rails_pulse.rb")
+        result = RailsPulse::Installers::ConfigUpdater.update(
+          destination: path,
+          source: File.join(self.class.source_root, "rails_pulse.rb")
+        )
+        @initializer_updated = result[:status] == :updated
+        return unless @initializer_updated
+
+        say "\nUpdated config/initializers/rails_pulse.rb with new settings:", :blue
+        result[:keys].each { |key| say "  - config.#{key}", :blue }
+        result[:hash_keys].each { |key| say "  - #{key}", :blue }
+        say "Review with git diff and keep or discard hunks.", :green
       end
 
-      def single_db_next_steps
-        [
-          "1. Run: rails db:migrate",
-          "2. Restart your Rails server"
-        ]
+      def warn_if_missing_schema_dump_false
+        return unless separate_database_missing_schema_dump_false?
+
+        say "\nIMPORTANT: Add schema_dump: false to the rails_pulse entry in", :yellow
+        say "config/database.yml. Without it, Rails may dump or load", :yellow
+        say "db/rails_pulse_structure.sql and db:migrate can fail with", :yellow
+        say "\"relation already exists\". Delete that structure file if present.", :yellow
       end
 
-      def separate_db_next_steps
-        [
-          "1. Run migrations for the rails_pulse database:",
-          "   rails db:migrate:rails_pulse",
-          "2. Restart your Rails server"
-        ]
+      def separate_database_missing_schema_dump_false?
+        config_path = File.join(root_path, "config/database.yml")
+        return false unless File.exist?(config_path)
+
+        require "yaml"
+        require "erb"
+        yaml_content = ERB.new(File.read(config_path)).result
+        db_config = YAML.safe_load(yaml_content, aliases: true)
+        pulse_entries = db_config.values.filter_map do |env|
+          env["rails_pulse"] if env.is_a?(Hash)
+        end
+        return false if pulse_entries.empty?
+
+        pulse_entries.any? { |entry| !entry.is_a?(Hash) || entry["schema_dump"] != false }
+      rescue
+        false
       end
 
       def offer_conversion_to_migrations
@@ -170,7 +294,11 @@ module RailsPulse
       def detect_missing_columns
         return {} unless rails_pulse_tables_exist?
 
-        connection = ActiveRecord::Base.connection
+        connection = if @is_separate_db && defined?(RailsPulse::ApplicationRecord)
+          RailsPulse::ApplicationRecord.connection
+        else
+          ActiveRecord::Base.connection
+        end
         missing = {}
 
         get_expected_schema_from_file.each do |table_name, columns|