rails_observatory 0.1.0

data/lib/rails_observatory/redis/time_series/query_builder.rb

@@ -0,0 +1,149 @@
+module RailsObservatory
+  class TimeSeries
+    class QueryBuilder
+      include Enumerable
+
+      def initialize(series_class)
+        @series_class = series_class
+        @conditions = {}
+        @samples = nil
+        @range_set = false
+        @group = nil
+        @range = (nil..)
+      end
+
+      def where(**conditions)
+        clone = self.clone
+        clone.instance_variable_set(:@conditions, @conditions.merge(conditions))
+        clone
+      end
+
+      def group(label)
+        clone = self.clone
+        clone.instance_variable_set(:@group, label)
+        clone
+      end
+
+      def slice(range)
+        clone = self.clone
+        clone.instance_variable_set(:@range_set, true)
+        clone.instance_variable_set(:@range, range)
+        clone
+      end
+
+      def downsample(samples, using:)
+        clone = self.clone
+        clone.instance_variable_set(:@samples, samples)
+        clone.instance_variable_set(:@agg_type, using)
+        clone
+      end
+
+
+
+      def sum
+        if @group
+          @agg_type = :sum
+          @samples = 1
+          to_a.index_by { _1.labels[@group] }.transform_values { _1.value }
+        else
+          raise "Cannot sum without grouping"
+        end
+      end
+
+      def avg
+        if @group
+          @agg_type = :avg
+          @samples = 1
+          to_a.index_by { _1.labels[@group] }
+        else
+          raise "Cannot avg without grouping"
+        end
+      end
+
+      def last
+        if @group
+          @agg_type = :last
+          @samples = 1
+          to_a.index_by { _1.labels[@group] }
+        else
+          raise "Cannot last without grouping"
+        end
+      end
+
+      def each
+        @range = ActiveSupport::IsolatedExecutionState[:observatory_slice] || (nil..) unless @range_set
+        agg_duration = build_agg_duration
+        mrange_args = ['TS.MRANGE', from_ts, to_ts, 'WITHLABELS']
+        mrange_args.push('LATEST') if @range.end.nil?
+        if @agg_type && @samples
+          if @range.end.present?
+            mrange_args.push("ALIGN", 'end')
+          elsif @range.begin.present?
+            mrange_args.push("ALIGN", 'start')
+          end
+          mrange_args.push("AGGREGATION", @agg_type.to_s.upcase, agg_duration, "EMPTY")
+        end
+        mrange_args.push('FILTER', *ts_filters)
+
+        # puts mrange_args.join(" ")
+        res = @series_class.redis.call(mrange_args)
+        return if res.nil?
+
+        res.each do |name, labels, data|
+          yield @series_class.new(
+            name:,
+            labels: Hash[*labels.flatten],
+            data:,
+            time_range: @range,
+            agg_duration: agg_duration
+          )
+        end
+      end
+
+      private
+
+      def build_agg_duration
+        end_time = @range.end || Time.now
+        start_time = @range.begin || 12.months.ago.to_time
+        available_datapoints = ((end_time - start_time) / 10.0).to_i
+        datapoints = [@samples, available_datapoints].min
+        ((end_time - start_time) * 1000 / datapoints).to_i
+      end
+
+      def from_ts
+        if @range.begin.nil?
+          "-"
+        else
+          @range.begin.to_i * 1000
+        end
+      end
+
+      def to_ts
+        if @range.end.nil?
+          "+"
+        else
+          @range.end.to_i * 1000
+        end
+      end
+
+      def ts_filters
+        raise 'Must specify name' if @conditions[:name].blank?
+
+        @conditions[@group] = "*" if @group
+        labels = @series_class.redis.call('SMEMBERS', "#{@conditions[:name]}:labels")
+        labels = labels.map { |l| [l.to_sym, nil] }.to_h
+        @conditions.reverse_merge!(labels)
+        @conditions.map do |k, v|
+          if v == "*"
+            "#{k}!="
+          elsif v.is_a? Array
+            "#{k}=(#{v.join(',')})"
+          else
+            "#{k}=#{v}"
+          end
+        end.to_a
+      end
+
+    end
+  end
+end

data/lib/rails_observatory/redis/time_series/timing_script.lua

@@ -0,0 +1,89 @@
+-- Helper function to get all combinations of a table
+local function generate_key_combinations(keys)
+  local n = #keys
+  local combs = {}
+  table.insert(combs, {})
+
+  local function helper(curr_comb, start_idx)
+    if start_idx <= n then
+      for i = start_idx, n do
+        local new_comb = {}
+        for _, v in ipairs(curr_comb) do
+            table.insert(new_comb, v)
+        end
+        table.insert(new_comb, keys[i])
+        table.insert(combs, new_comb)
+        helper(new_comb, i + 1)
+      end
+    end
+  end
+
+  helper({}, 1)
+  return combs
+end
+
+local function extract_parent_label(base_name)
+  local index = string.find(base_name, "/")
+  if index then
+    return string.sub(base_name, 1, index - 1)
+  else
+    return nil
+  end
+end
+
+-- Main script begins here
+local metric_name = tostring(ARGV[1])  -- Ensure it's a string
+local value_to_add = tonumber(ARGV[2]) -- Ensure it's a number
+local raw_retention = 10000 -- Hardcoded to 10ms
+local compaction_retention = 31536000000 -- Hardcoded to 1 year in ms (365*24*60*60*1000)
+local compactions = {"avg", "min", "max"}
+
+-- Assuming base_name is defined somewhere above
+local parent_label = extract_parent_label(metric_name)
+
+-- Extracting labels
+local labels = {}
+local keys = {}
+for i=3, #ARGV, 2 do
+  local key = tostring(ARGV[i])
+  local value = tostring(ARGV[i+1])
+  labels[key] = value
+  redis.call("SADD", metric_name .. ':labels', key)
+  table.insert(keys, key)
+end
+
+local key_combinations = generate_key_combinations(keys)
+
+-- For each combination, upsert and add labels
+for _, comb_keys in ipairs(key_combinations) do
+  local ts_name = metric_name
+  local label_set = {}
+
+  if parent_label then
+    table.insert(label_set, "parent")
+    table.insert(label_set, parent_label)
+  end
+
+  for _, key in ipairs(comb_keys) do
+    ts_name = ts_name .. ":" .. labels[key]
+    table.insert(label_set, key)
+    table.insert(label_set, labels[key])
+  end
+
+  if redis.call("EXISTS", ts_name) == 0 then
+    redis.call("TS.CREATE", ts_name, "RETENTION", raw_retention, "CHUNK_SIZE", 48)
+  end
+
+  -- Handle the compactions (avg, min, max)
+  for _, compaction in ipairs(compactions) do
+    local compaction_key = ts_name .. "_" .. compaction
+    if redis.call("EXISTS", compaction_key) == 0 then
+      redis.call("TS.CREATE", compaction_key, "RETENTION", compaction_retention, "CHUNK_SIZE", 48, "LABELS","name", metric_name, "compaction", compaction, unpack(label_set))
+      redis.call("TS.CREATERULE", ts_name, compaction_key, "AGGREGATION", compaction, 10000)
+      return redis.call("TS.INFO", compaction_key)
+    end
+  end
+  redis.call("TS.ADD", ts_name, "*", value_to_add)
+end
+
+return "OK"

data/lib/rails_observatory/redis/time_series.rb

@@ -0,0 +1,91 @@
+require_relative 'time_series/insertion'
+require_relative 'time_series/query_builder'
+
+module RailsObservatory
+  class TimeSeries
+
+    extend Insertion
+
+    attr_reader :labels, :name, :data
+
+    def self.redis
+      Rails.configuration.rails_observatory.redis
+    end
+
+    def redis
+      self.class.redis
+    end
+
+    def self.with_slice(time_range)
+      ActiveSupport::IsolatedExecutionState[:observatory_slice] = time_range
+      yield
+    ensure
+      ActiveSupport::IsolatedExecutionState[:observatory_slice] = nil
+    end
+
+    def self.where(**conditions)
+      QueryBuilder.new(self).where(**conditions)
+    end
+
+    def initialize(name:, labels: {}, data:, time_range:, agg_duration:)
+      @name = name
+      @time_range = time_range
+      @agg_duration = agg_duration
+      @labels = labels.deep_symbolize_keys
+      @data = data
+    end
+
+    def start_time
+      @time_range.begin.nil? ? Time.utc(2023, 1, 1, 0, 0, 0) : @time_range.begin
+    end
+
+    def start_time_ms
+      start_time.to_i.in_milliseconds
+    end
+
+    def end_time_ms
+      end_time.to_i.in_milliseconds
+    end
+
+    def end_time
+      @time_range.end.nil? ? Time.now : @time_range.end
+    end
+
+    def filled_data
+      if @time_range.end.nil?
+        Enumerator
+          .produce(to_ms(start_time)) { |t| t + @agg_duration }
+          .take_while { |t| t < to_ms(end_time) }
+          .map do |t|
+          match = data.find { |ts, _| ts == t }
+          if match
+            timestamp, val = match
+            [timestamp, val.to_f]
+          else
+            [t, 0]
+          end
+        end
+      else
+        data.map do |ts, value|
+          [ts, value.to_i || 0]
+        end
+      end
+    end
+
+    def empty?
+      data.empty?
+    end
+
+    def value
+      data.dig(0, 1).to_i
+    end
+
+    def to_ms(duration)
+      self.class.to_ms(duration)
+    end
+
+    def self.to_ms(duration)
+      duration.to_i * 1_000
+    end
+  end
+end

data/lib/rails_observatory/serializers/event_serializer.rb

@@ -0,0 +1,19 @@
+module RailsObservatory
+  class EventSerializer
+    def serialize(event)
+        {
+          name: event.name,
+          payload: Serializer.serialize(event.payload),
+          start_at: event.time,
+          end_at: event.end,
+          duration: event.duration,
+          allocations: event.allocations,
+          failed: event.payload.include?(:exception),
+        }
+    end
+
+    def self.klass
+      ActiveSupport::Notifications::Event
+    end
+  end
+end

data/lib/rails_observatory/serializers/headers_serializer.rb

@@ -0,0 +1,12 @@
+module RailsObservatory
+  class HeadersSerializer
+    def serialize(headers)
+      http_headers = Hash[*headers.select { |k, v| k.start_with?("HTTP_") }.flatten]
+      http_headers.transform_keys! { |k| k.sub("HTTP_", "").downcase.capitalize.dasherize }
+    end
+
+    def self.klass
+      ActionDispatch::Http::Headers
+    end
+  end
+end

data/lib/rails_observatory/serializers/job_serializer.rb

@@ -0,0 +1,11 @@
+module RailsObservatory
+  class JobSerializer
+    def serialize(job)
+      { class: job.class.name, job_id: job.job_id, queue_name: job.queue_name }
+    end
+
+    def self.klass
+      ActiveJob::Base
+    end
+  end
+end

data/lib/rails_observatory/serializers/mail_delivery_job_serializer.rb

@@ -0,0 +1,14 @@
+module RailsObservatory
+  class MailDeliveryJobSerializer < JobSerializer
+    def serialize(job)
+      super.merge(
+        mailer_class: job.arguments.first,
+        mailer_method: job.arguments.second
+      )
+    end
+
+    def self.klass
+      ActionMailer::MailDeliveryJob
+    end
+  end
+end

data/lib/rails_observatory/serializers/request_serializer.rb

@@ -0,0 +1,17 @@
+module RailsObservatory
+  class RequestSerializer
+    def serialize(request)
+      {
+        method: request.method,
+        path: request.path,
+        format: request.format,
+        route_pattern: request.route_uri_pattern,
+        headers: Serializer.serialize(request.headers),
+      }
+    end
+
+    def self.klass
+      ActionDispatch::Request
+    end
+  end
+end

data/lib/rails_observatory/serializers/response_serializer.rb

@@ -0,0 +1,14 @@
+module RailsObservatory
+  class ResponseSerializer
+    def serialize(response)
+      {
+        status: response.status,
+        # headers: Serializer.serialize(response.headers),
+      }
+    end
+
+    def self.klass
+      ActionDispatch::Response
+    end
+  end
+end

data/lib/rails_observatory/serializers/serializer.rb

@@ -0,0 +1,51 @@
+require_relative './job_serializer'
+require_relative './mail_delivery_job_serializer'
+require_relative './event_serializer'
+require_relative './request_serializer'
+require_relative './headers_serializer'
+require_relative './response_serializer'
+
+module RailsObservatory
+  class Serializer
+
+    PERMITTED_TYPES = [NilClass, String, Integer, Float, TrueClass, FalseClass]
+
+    ADDITIONAL_SERIALIZERS = [JobSerializer, MailDeliveryJobSerializer, EventSerializer, RequestSerializer, HeadersSerializer, ResponseSerializer]
+
+    class << self
+
+
+      def serialize(argument)
+        serialize_payload(argument)
+      end
+
+      def serialize_payload(argument)
+        case argument
+        when *PERMITTED_TYPES
+          argument
+        when Array
+          argument.map { serialize_payload(_1) }
+        when ActiveSupport::HashWithIndifferentAccess
+          serialize_hash(argument)
+        when Hash
+          serialize_hash(argument)
+        when -> (arg) { arg.respond_to?(:permitted?) && arg.respond_to?(:to_h) }
+          serialize_hash(argument.to_h)
+        when Symbol
+          argument.to_s
+        else
+          ADDITIONAL_SERIALIZERS.find { argument.is_a?(_1.klass) }&.new&.serialize(argument) || "Unable to serialize #{argument.class.name}"
+        end
+      end
+
+      def serialize_hash(argument)
+        argument.each_with_object({}) do |(key, value), hash|
+          case key
+          when String, Symbol
+            hash[key] = serialize_payload(value)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rails_observatory/version.rb

@@ -0,0 +1,3 @@
+module RailsObservatory
+  VERSION = "0.1.0"
+end

data/lib/rails_observatory.rb

@@ -0,0 +1,3 @@
+require 'rails_observatory/engine'
+module RailsObservatory
+end