rails_pulse 0.4.2 → 0.5.0.pre.2

data/app/services/rails_pulse/cloud/route_pattern.rb

@@ -0,0 +1,111 @@
+module RailsPulse
+  module Cloud
+    # The path a recognised route is sent to Cloud under: its pattern as
+    # written in the host's routes, never a value from a request.
+    #
+    # The stored path is the request path with each parameter value swapped
+    # back for its name, which leaves a value in place when two parameters
+    # share it (`/users/5/posts/5`) or when a glob captures several segments
+    # (`get "*slug"`). So the stored path is matched against the router's
+    # patterns for the same controller action, and the pattern is sent.
+    # When no pattern lines up (a route since removed, or one inside a
+    # mounted engine), each literal segment that does not look like code is
+    # replaced with `*`; a segment without a letter is taken for an ID.
+    #
+    #   patterns = RoutePattern.new
+    #   patterns.path_for("/users/5/posts/5", "posts#show")  # => "/users/:user_id/posts/:id"
+    #   patterns.path_for("/docs/getting-started", "pages#show")  # => "/*slug" for `get "*slug"`
+    class RoutePattern
+      PARAMETER = /\A:[A-Za-z_]\w*(?:\.[A-Za-z0-9]+)?\z/
+      # More optional groups than this in one route are not expanded.
+      MAX_VARIANTS = 16
+
+      def initialize(routes = Rails.application.routes.routes)
+        @routes = routes
+      end
+
+      def path_for(path, controller_action)
+        stored = segments(path.to_s)
+        patterns_for(controller_action).each do |pattern|
+          matched = match(pattern, stored)
+          return matched if matched
+        end
+        fallback(stored)
+      end
+
+      private
+
+      # { "posts#show" => [["", "posts", ":id"], ...] }, built once per instance.
+      def patterns_for(controller_action)
+        @patterns ||= @routes.each_with_object(Hash.new { |hash, key| hash[key] = [] }) do |route, index|
+          controller = route.defaults[:controller]
+          action = route.defaults[:action]
+          next unless controller && action
+
+          variants(route.path.spec.to_s).each { |spec| index["#{controller}##{action}"] << segments(spec) }
+        end
+        @patterns.fetch(controller_action.to_s, [])
+      end
+
+      # "/posts(/:page)(.:format)" => ["/posts/:page", "/posts"], with any
+      # `.:format` dropped because the stored path keeps the extension on its
+      # last segment rather than as a separate one.
+      def variants(spec)
+        spec = spec.gsub("(.:format)", "")
+        expanded = [ spec ]
+        while (template = expanded.find { |candidate| candidate.include?("(") })
+          break if expanded.size > MAX_VARIANTS
+
+          expanded.delete(template)
+          expanded.push(*optional_group(template))
+        end
+        expanded.reject { |candidate| candidate.include?("(") }.uniq
+      end
+
+      # The template with its innermost optional group kept and removed.
+      def optional_group(template)
+        start = template.rindex("(")
+        finish = template.index(")", start)
+        return [ template.delete("()") ] unless finish
+
+        before = template[0...start]
+        after = template[(finish + 1)..]
+        [ before + template[(start + 1)...finish] + after, before + after ]
+      end
+
+      def segments(path)
+        path.split("/", -1)
+      end
+
+      def match(pattern, stored)
+        result = []
+        pattern.each_with_index do |expected, index|
+          if expected.start_with?("*")
+            return nil if index >= stored.size
+
+            result << expected
+            return result.join("/")
+          end
+
+          actual = stored[index]
+          return nil if actual.nil?
+          return nil unless expected == actual || expected.include?(":")
+
+          result << expected
+        end
+
+        stored.size == pattern.size ? result.join("/") : nil
+      end
+
+      def fallback(stored)
+        stored.map do |segment|
+          next segment if segment.empty?
+          next segment if segment.match?(PARAMETER)
+          next segment if PathPrefix.safe_segment?(segment) && segment.match?(/[A-Za-z]/)
+
+          "*"
+        end.join("/")
+      end
+    end
+  end
+end

data/app/services/rails_pulse/cloud/summary_items.rb

@@ -0,0 +1,188 @@
+module RailsPulse
+  module Cloud
+    # The `summary` items for one hour, built from that hour's
+    # rails_pulse_summaries rows. Nothing is returned for an hour SummaryJob
+    # has not written, so a partial hour is never sent.
+    #
+    # Each item names its subject by content (a route's pattern, a query's
+    # hash, a job's class), never by a local id, and carries no field the
+    # sync contract does not list. Route and query tags are not sent: they
+    # are free-form and can name customers.
+    class SummaryItems
+      ROUTE_LIMIT = 2_000
+      QUERY_LIMIT = 2_000
+      UNMATCHED_LIMIT = 20
+      CALL_SITES = 2
+
+      # The key of the item that combines everything past a limit.
+      OTHER = "*".freeze
+
+      DURATIONS = %i[
+        avg_duration min_duration max_duration p50_duration p95_duration p99_duration stddev_duration total_duration
+      ].freeze
+      REQUEST_COUNTS = %i[error_count status_2xx status_3xx status_4xx status_5xx].freeze
+      JOB_COUNTS = %i[error_count success_count].freeze
+
+      # @param period_start [Time] the start of an hour, in the application's zone
+      # @param adapter [String] the Rails Pulse database adapter, which decides
+      #   how a double-quoted SQL token is read
+      def initialize(period_start, adapter:, route_patterns: RoutePattern.new, limits: {})
+        @period_start = period_start
+        @adapter = adapter
+        @route_patterns = route_patterns
+        @route_limit = limits.fetch(:route, ROUTE_LIMIT)
+        @query_limit = limits.fetch(:query, QUERY_LIMIT)
+        @unmatched_limit = limits.fetch(:unmatched, UNMATCHED_LIMIT)
+      end
+
+      def items
+        return [] unless summarized?
+
+        requests_items + route_items + unmatched_items + query_items + job_items + exception_items
+      end
+
+      def summarized?
+        rows_of("RailsPulse::Request").any? { |row| row.summarizable_id.zero? }
+      end
+
+      private
+
+      def rows
+        @rows ||= Summary.where(period_type: "hour", period_start: @period_start).to_a.group_by(&:summarizable_type)
+      end
+
+      def rows_of(type)
+        rows.fetch(type, [])
+      end
+
+      def item(kind, fields)
+        { type: "summary", kind: kind, period_start: @period_start.utc.iso8601, **fields }
+      end
+
+      def metrics(row_or_rows, counts)
+        group = Array(row_or_rows)
+        if group.size == 1
+          row = group.first
+          { count: row[:count].to_i, **DURATIONS.index_with { |column| row[column]&.to_f }, **counts.index_with { |column| row[column].to_i } }
+        else
+          CombinedMetrics.of(group, counts: counts)
+        end
+      end
+
+      # -- Requests and routes -------------------------------------------
+
+      def requests_items
+        overall = rows_of("RailsPulse::Request").find { |row| row.summarizable_id.zero? }
+        [ item("requests", metrics(overall, REQUEST_COUNTS)) ]
+      end
+
+      # Route rows split into recognised routes and requests the router did
+      # not match, which are stored without a controller action.
+      def route_rows
+        @route_rows ||= begin
+          route_summaries = rows_of("RailsPulse::Route").select { |row| row.count.to_i.positive? }
+          routes = Route.where(id: route_summaries.map(&:summarizable_id)).index_by(&:id)
+          route_summaries.filter_map do |row|
+            route = routes[row.summarizable_id]
+            [ route, row ] if route
+          end.partition { |route, _| route.controller_action.present? }
+        end
+      end
+
+      def route_items
+        groups = route_rows.first.group_by do |route, _|
+          [ @route_patterns.path_for(route.path, route.controller_action), route.controller_action ]
+        end
+
+        ranked = groups.sort_by { |key, members| [ -members.sum { |_, row| row.total_duration.to_f }, key.map(&:to_s) ] }
+        kept, rest = ranked.first(@route_limit), ranked.drop(@route_limit)
+
+        items = kept.map do |(path, controller_action), members|
+          methods = members.flat_map { |route, _| route.http_methods_list }.uniq.sort
+          item("route", path: path, controller_action: controller_action, http_methods: methods,
+                        **metrics(members.map(&:last), REQUEST_COUNTS))
+        end
+        items << item("route", path: OTHER, controller_action: nil, http_methods: [], **metrics(rest.flat_map { |_, members| members.map(&:last) }, REQUEST_COUNTS)) if rest.any?
+        items
+      end
+
+      def unmatched_items
+        groups = route_rows.last.group_by { |route, _| PathPrefix.for(route.path) }
+        ranked = groups.sort_by { |prefix, members| [ -members.sum { |_, row| row.count.to_i }, prefix ] }
+        kept, rest = ranked.first(@unmatched_limit), ranked.drop(@unmatched_limit)
+
+        items = kept.map { |prefix, members| item("unmatched", path_prefix: prefix, **metrics(members.map(&:last), REQUEST_COUNTS)) }
+        items << item("unmatched", path_prefix: PathPrefix::OTHER, **metrics(rest.flat_map { |_, members| members.map(&:last) }, REQUEST_COUNTS)) if rest.any?
+        items
+      end
+
+      # -- Queries -------------------------------------------------------
+
+      def query_items
+        query_summaries = rows_of("RailsPulse::Query").select { |row| row.count.to_i.positive? }
+        queries = Query.where(id: query_summaries.map(&:summarizable_id)).index_by(&:id)
+        ranked = query_summaries.select { |row| queries[row.summarizable_id] }.sort_by { |row| [ -row.total_duration.to_f, row.summarizable_id ] }
+        kept, rest = ranked.first(@query_limit), ranked.drop(@query_limit)
+        sites = call_sites(kept.map(&:summarizable_id))
+
+        items = kept.map do |row|
+          query = queries[row.summarizable_id]
+          shape = QueryShape.for_adapter(query.normalized_sql, @adapter)
+          item("query", hashed_sql: query.hashed_sql, label: shape.label, sql_shape: shape.sql_shape,
+                        call_sites: sites.fetch(query.id, []), **metrics(row, []))
+        end
+        items << item("query", hashed_sql: OTHER, label: "Other queries", sql_shape: nil, call_sites: [], **metrics(rest, [])) if rest.any?
+        items
+      end
+
+      # The two code locations that ran each query most often in the hour.
+      # Only paths inside the application are sent: a location outside it is
+      # stored as an absolute path, which can name the server's directories.
+      def call_sites(query_ids)
+        return {} if query_ids.empty?
+
+        Operation
+          .where(occurred_at: @period_start...(@period_start + 1.hour), query_id: query_ids)
+          .where.not(codebase_location: nil)
+          .group(:query_id, :codebase_location)
+          .count
+          .select { |(_, location), _| application_path?(location) }
+          .group_by { |(query_id, _), _| query_id }
+          .transform_values do |entries|
+            entries.sort_by { |(_, location), count| [ -count, location ] }.first(CALL_SITES).map { |(_, location), _| location }
+          end
+      end
+
+      def application_path?(location)
+        location.present? && !location.start_with?("/") && !location.include?("..") && !location.match?(/\A[A-Za-z]:/)
+      end
+
+      # -- Jobs and exceptions ---------------------------------------------
+
+      def job_items
+        job_summaries = rows_of("RailsPulse::Job").select { |row| row.count.to_i.positive? }
+        jobs = Job.where(id: job_summaries.map(&:summarizable_id)).index_by(&:id)
+        job_summaries.filter_map do |row|
+          job = jobs[row.summarizable_id]
+          next unless job
+
+          item("job", name: job.name, queue_name: job.queue_name, **metrics(row, JOB_COUNTS))
+        end
+      end
+
+      # Exceptions have no duration, so these items carry only a count.
+      def exception_items
+        summaries = rows_of("RailsPulse::ExceptionGroup")
+        overall, per_group = summaries.partition { |row| row.summarizable_id.zero? }
+        groups = ExceptionGroup.where(id: per_group.map(&:summarizable_id)).pluck(:id, :fingerprint).to_h
+
+        items = per_group.filter_map do |row|
+          fingerprint = groups[row.summarizable_id]
+          item("exception_group", fingerprint: fingerprint, count: row.count.to_i) if fingerprint
+        end
+        overall.each { |row| items << item("exceptions", count: row.count.to_i) }
+        items
+      end
+    end
+  end
+end

data/app/services/rails_pulse/cloud/sync.rb

@@ -0,0 +1,275 @@
+module RailsPulse
+  module Cloud
+    # Sends this installation's data to Rails Pulse Cloud by the sync
+    # contract. `hourly` runs after SummaryJob: it queues each newly
+    # summarized hour with the exception groups and deployments that changed,
+    # then sends what is due. `minutely` sends the health update for the
+    # minute just ended, queues deployments recorded or finished since the
+    # last run, and sends what is due, which is also how retries happen.
+    #
+    # Batches wait in BufferedBatch until Cloud accepts them; health updates
+    # are never buffered, because one that arrives late is no use. With
+    # config.cloud unset nothing here runs, and no failure here is raised to
+    # the caller: Cloud being down, or refusing the key, must never affect
+    # the host application or local tracking.
+    class Sync
+      # Hours queued in one run, so catching up after an outage is spread
+      # over several.
+      MAX_HOURS_PER_RUN = 24
+      MAX_BATCHES_PER_FLUSH = 50
+
+      BACKOFF_START = 1.minute
+      BACKOFF_MAX = 1.hour
+      DEFAULT_RETRY_AFTER = 1.minute
+      # After Cloud refuses the key, the plan or the application, buffering
+      # continues and sending is retried this often.
+      REFUSED_RETRY = 1.hour
+      UNSUPPORTED_RETRY = 1.day
+
+      DEPRECATION_HEADER = "Rails-Pulse-Contract-Deprecated".freeze
+
+      # The hours and record windows the next hourly run would queue,
+      # without queueing anything or creating the installation row.
+      Plan = Struct.new(:hours, :exception_groups, :deployments, keyword_init: true)
+
+      def initialize(settings: RailsPulse.configuration.cloud, client: nil, now: Time.current, jitter: nil)
+        @settings = settings
+        @client = client || Client.new(settings)
+        @now = now
+        @jitter = jitter || ->(delay) { delay * (0.5 + (rand / 2)) }
+      end
+
+      def enabled?
+        @settings.enabled? && tables_available?
+      end
+
+      def hourly
+        guarded("hourly sync") do
+          plan = plan_for(installation)
+          route_patterns = RoutePattern.new
+          plan.hours.each do |hour|
+            items = SummaryItems.new(hour, adapter: envelope[:database_adapter], route_patterns: route_patterns).items
+            queue(items) { installation.update!(last_hour_sent_at: hour) }
+          end
+          queue_records(plan, exception_groups: true)
+          flush
+        end
+      end
+
+      def minutely
+        guarded("health update") do
+          send_health
+          queue_records(plan_for(installation), exception_groups: false)
+          flush
+        end
+      end
+
+      # What the next hourly run would queue for `record`, which may be nil
+      # before the first sync.
+      def plan_for(record)
+        Plan.new(hours: unsent_hours(record&.last_hour_sent_at),
+                 exception_groups: window_since(record&.exception_groups_sent_through),
+                 deployments: window_since(record&.deployments_sent_through))
+      end
+
+      def envelope
+        @envelope ||= Batch.envelope(application: @settings.application, environment: @settings.environment,
+                                     installation_id: installation.installation_id)
+      end
+
+      private
+
+      def installation
+        @installation ||= Installation.current
+      end
+
+      def guarded(what)
+        return :disabled unless enabled?
+
+        yield
+        :ok
+      rescue StandardError => e
+        RailsPulse.logger.error("Rails Pulse Cloud #{what} failed: #{e.class} - #{e.message}")
+        record_failure("#{what} failed: #{e.class}: #{e.message}")
+        :failed
+      end
+
+      # The failure is already logged; if the database is what failed, it
+      # cannot be recorded there too.
+      def record_failure(message)
+        installation.record_error!(message, now: @now)
+      rescue StandardError
+        nil
+      end
+
+      def tables_available?
+        Installation.table_exists? && BufferedBatch.table_exists?
+      rescue ActiveRecord::ActiveRecordError
+        false
+      end
+
+      # -- Queueing ------------------------------------------------------
+
+      # Summarized hours after the last one queued, oldest first. The first
+      # sync starts at the latest hour rather than replaying history.
+      def unsent_hours(last_sent)
+        heartbeats = Summary.overall_requests.for_period_type("hour")
+        latest = heartbeats.maximum(:period_start)
+        return [] unless latest
+
+        from = last_sent ? last_sent + 1.second : latest
+        heartbeats.where(period_start: from..latest).order(:period_start).limit(MAX_HOURS_PER_RUN)
+          .pluck(:period_start).map(&:in_time_zone)
+      end
+
+      # Rows changed since the cursor and before now. The first sync covers
+      # the hourly summary retention, so Cloud can name what those hours count.
+      def window_since(cursor)
+        (cursor || (@now - RailsPulse.configuration.hourly_summary_retention))...@now
+      end
+
+      def queue_records(plan, exception_groups:)
+        items = []
+        items += RecordItems.exception_groups_updated(plan.exception_groups) if exception_groups
+        items += RecordItems.deployments_updated(plan.deployments)
+        cursors = { deployments_sent_through: plan.deployments.end }
+        cursors[:exception_groups_sent_through] = plan.exception_groups.end if exception_groups
+        queue(items) { installation.update!(cursors) }
+      end
+
+      # Batches the items into the buffer and moves the cursor in the same
+      # transaction, so a crash between the two neither loses nor repeats them.
+      def queue(items)
+        RailsPulse::ApplicationRecord.transaction do
+          Batch.build(items, envelope: envelope).each { |batch| BufferedBatch.enqueue!(batch, now: @now) }
+          yield
+        end
+        dropped = BufferedBatch.prune!(@now)
+        RailsPulse.logger.warn("Rails Pulse Cloud buffer full: dropped #{dropped} oldest batch(es)") if dropped.positive?
+      end
+
+      # -- Sending -------------------------------------------------------
+
+      def flush
+        return if installation.paused?(@now)
+
+        # Read one at a time: a split adds batches that are due at once.
+        MAX_BATCHES_PER_FLUSH.times do
+          buffered = BufferedBatch.due(@now).first
+          break unless buffered && deliver(buffered) == :continue
+        end
+      end
+
+      def deliver(buffered)
+        contents = buffered.contents.merge("sent_at" => @now.utc.iso8601)
+        response = @client.post(contents.to_json)
+
+        case outcome(response)
+        when :accepted
+          buffered.destroy!
+          installation.record_success!(deprecated_on: response.header(DEPRECATION_HEADER), now: @now)
+          :continue
+        when :malformed
+          buffered.destroy!
+          RailsPulse.logger.error("Rails Pulse Cloud refused batch #{buffered.batch_id} as malformed: #{response.body}")
+          installation.record_error!("Cloud refused a batch as malformed and it was dropped: #{response.message}", now: @now)
+          :continue
+        when :too_large
+          split(buffered, contents)
+          :continue
+        when :rate_limited
+          retry_after = response.header("Retry-After").to_i
+          buffered.update!(next_attempt_at: @now + (retry_after.positive? ? retry_after : DEFAULT_RETRY_AFTER),
+                           last_error: response.message)
+          :stop
+        when :refused, :unsupported
+          pause(response)
+          :stop
+        else
+          back_off(buffered, response.message)
+          :stop
+        end
+      rescue Client::Unavailable => e
+        back_off(buffered, e.message)
+        :stop
+      end
+
+      def send_health
+        return if installation.paused?(@now)
+
+        batch = Batch.new([ HealthItem.for_minute_before(@now) ], envelope: envelope, sent_at: @now)
+        response = @client.post(batch.to_json)
+        case outcome(response)
+        when :accepted
+          installation.record_success!(deprecated_on: response.header(DEPRECATION_HEADER), now: @now)
+          installation.update!(last_health_at: @now)
+        when :refused, :unsupported
+          pause(response)
+        else
+          installation.record_error!("Health update not sent: #{response.message}", now: @now)
+        end
+      rescue Client::Unavailable => e
+        installation.record_error!("Health update not sent: #{e.message}", now: @now)
+      end
+
+      def outcome(response)
+        case response.status
+        when 202 then :accepted
+        when 200 then :accepted
+        when 400 then :malformed
+        when 401, 402, 404 then :refused
+        when 413 then :too_large
+        when 422 then :unsupported
+        when 429 then :rate_limited
+        else :unavailable
+        end
+      end
+
+      def pause(response)
+        if response.status == 422
+          installation.pause!(@now + UNSUPPORTED_RETRY,
+                              "Rails Pulse Cloud does not accept this gem's contract version (#{response.message}). Upgrade the rails_pulse gem.",
+                              now: @now)
+        else
+          installation.pause!(@now + REFUSED_RETRY, "#{refusal(response.status)}: #{response.message}", now: @now)
+        end
+      end
+
+      def refusal(status)
+        case status
+        when 401 then "Rails Pulse Cloud refused config.cloud.api_key"
+        when 402 then "The Rails Pulse Cloud subscription is not active"
+        else "Rails Pulse Cloud does not know the application #{@settings.application.inspect}"
+        end
+      end
+
+      # 1 minute, doubling, up to an hour, with jitter so installations that
+      # failed together do not retry together.
+      def back_off(buffered, message)
+        attempts = buffered.attempts + 1
+        delay = [ BACKOFF_START * (2**(attempts - 1)), BACKOFF_MAX ].min
+        buffered.update!(attempts: attempts, next_attempt_at: @now + @jitter.call(delay.to_f).seconds, last_error: message)
+        installation.record_error!("Cloud unavailable, retrying: #{message}", now: @now)
+      end
+
+      # Halves a batch Cloud found too large. Each half is a new batch with
+      # its own id: it carries different items, so it is not a resend.
+      def split(buffered, contents)
+        items = contents["items"]
+        if items.size <= 1
+          buffered.destroy!
+          installation.record_error!("Cloud refused a single item as too large and it was dropped", now: @now)
+          return
+        end
+
+        envelope = contents.except("batch_id", "sent_at", "items").transform_keys(&:to_sym)
+        RailsPulse::ApplicationRecord.transaction do
+          items.each_slice((items.size / 2.0).ceil) do |half|
+            BufferedBatch.enqueue!(Batch.new(half, envelope: envelope), now: @now)
+          end
+          buffered.destroy!
+        end
+      end
+    end
+  end
+end