rubydb 0.1.0

data/lib/rubydb/monitoring/metrics.rb

@@ -0,0 +1,359 @@
+# frozen_string_literal: true
+
+require "time"
+require "thread"
+require "json"
+require "monitor"
+
+module RubyDB
+  module Monitoring
+    # Metrics - Collects and manages metrics
+    class Metrics
+      attr_reader :stats
+
+      # Metric types
+      TYPE_COUNTER = :counter
+      TYPE_GAUGE = :gauge
+      TYPE_HISTOGRAM = :histogram
+      TYPE_SUMMARY = :summary
+
+      def initialize(config = {})
+        @config = config
+        @metrics = {}
+        # Mutating helpers call counter/gauge/histogram, which also acquire
+        # the lock. A reentrant monitor prevents self-deadlock while keeping
+        # metric updates atomic across threads.
+        @lock = Monitor.new
+        @collectors = []
+        @flush_interval = config[:flush_interval] || 60
+        @retention_period = config[:retention_period] || 3600
+        @stats = {
+          metrics_count: 0,
+          samples_collected: 0,
+          flushes: 0,
+          errors: 0
+        }
+        @flush_thread = nil
+        @running = false
+        @storage = []
+
+        start_flush_thread if config[:auto_flush] != false
+      end
+
+      def counter(name, labels = {})
+        @lock.synchronize do
+          key = metric_key(name, labels)
+          @metrics[key] ||= {
+            type: TYPE_COUNTER,
+            name: name,
+            labels: labels,
+            value: 0,
+            created_at: Time.now,
+            updated_at: Time.now
+          }
+          @metrics[key]
+        end
+      end
+
+      def gauge(name, labels = {})
+        @lock.synchronize do
+          key = metric_key(name, labels)
+          @metrics[key] ||= {
+            type: TYPE_GAUGE,
+            name: name,
+            labels: labels,
+            value: 0,
+            created_at: Time.now,
+            updated_at: Time.now
+          }
+          @metrics[key]
+        end
+      end
+
+      def histogram(name, labels = {}, buckets = nil)
+        @lock.synchronize do
+          key = metric_key(name, labels)
+          @metrics[key] ||= {
+            type: TYPE_HISTOGRAM,
+            name: name,
+            labels: labels,
+            buckets: buckets || [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
+            values: [],
+            created_at: Time.now,
+            updated_at: Time.now
+          }
+          @metrics[key]
+        end
+      end
+
+      def summary(name, labels = {})
+        @lock.synchronize do
+          key = metric_key(name, labels)
+          @metrics[key] ||= {
+            type: TYPE_SUMMARY,
+            name: name,
+            labels: labels,
+            values: [],
+            created_at: Time.now,
+            updated_at: Time.now
+          }
+          @metrics[key]
+        end
+      end
+
+      def increment(name, labels = {}, amount = 1)
+        @lock.synchronize do
+          metric = counter(name, labels)
+          metric[:value] += amount
+          metric[:updated_at] = Time.now
+          @stats[:samples_collected] += 1
+          metric[:value]
+        end
+      end
+
+      def decrement(name, labels = {}, amount = 1)
+        @lock.synchronize do
+          metric = counter(name, labels)
+          metric[:value] -= amount
+          metric[:updated_at] = Time.now
+          @stats[:samples_collected] += 1
+          metric[:value]
+        end
+      end
+
+      def set_gauge(name, labels = {}, value)
+        @lock.synchronize do
+          metric = gauge(name, labels)
+          metric[:value] = value
+          metric[:updated_at] = Time.now
+          @stats[:samples_collected] += 1
+          metric[:value]
+        end
+      end
+
+      def observe_histogram(name, labels = {}, value)
+        @lock.synchronize do
+          metric = histogram(name, labels)
+          metric[:values] << value
+          metric[:updated_at] = Time.now
+          @stats[:samples_collected] += 1
+          metric[:values].size
+        end
+      end
+
+      def observe_summary(name, labels = {}, value)
+        @lock.synchronize do
+          metric = summary(name, labels)
+          metric[:values] << value
+          metric[:updated_at] = Time.now
+          @stats[:samples_collected] += 1
+          metric[:values].size
+        end
+      end
+
+      def get(name, labels = {})
+        @lock.synchronize do
+          key = metric_key(name, labels)
+          @metrics[key]
+        end
+      end
+
+      def get_all
+        @lock.synchronize do
+          @metrics.dup
+        end
+      end
+
+      def get_by_name(name)
+        @lock.synchronize do
+          @metrics.select { |key, _| key.start_with?(name.to_s) }
+        end
+      end
+
+      def reset(name, labels = {})
+        @lock.synchronize do
+          key = metric_key(name, labels)
+          metric = @metrics[key]
+          if metric
+            case metric[:type]
+            when TYPE_COUNTER
+              metric[:value] = 0
+            when TYPE_GAUGE
+              metric[:value] = 0
+            when TYPE_HISTOGRAM, TYPE_SUMMARY
+              metric[:values] = []
+            end
+            metric[:updated_at] = Time.now
+          end
+          metric
+        end
+      end
+
+      def clear
+        @lock.synchronize do
+          @metrics.clear
+          @storage.clear
+        end
+      end
+
+      def flush
+        @lock.synchronize do
+          return if @metrics.empty?
+
+          snapshot = {
+            timestamp: Time.now.iso8601,
+            metrics: @metrics.dup
+          }
+
+          @storage << snapshot
+          @stats[:flushes] += 1
+
+          # Trim old data
+          cutoff = Time.now - @retention_period
+          @storage.delete_if { |s| Time.parse(s[:timestamp]) < cutoff }
+
+          snapshot
+        end
+      end
+
+      def query(start_time = nil, end_time = nil, name = nil)
+        @lock.synchronize do
+          start_time = Time.parse(start_time) if start_time.is_a?(String)
+          end_time = Time.parse(end_time) if end_time.is_a?(String)
+
+          results = @storage.select do |snapshot|
+            time = Time.parse(snapshot[:timestamp])
+            (start_time.nil? || time >= start_time) &&
+            (end_time.nil? || time <= end_time)
+          end
+
+          if name
+            results = results.map do |snapshot|
+              {
+                timestamp: snapshot[:timestamp],
+                metrics: snapshot[:metrics].select { |key, _| key.start_with?(name.to_s) }
+              }
+            end
+          end
+
+          results
+        end
+      end
+
+      def add_collector(collector)
+        @collectors << collector
+      end
+
+      def collect
+        @lock.synchronize do
+          @collectors.each do |collector|
+            begin
+              collector.call(self)
+            rescue => e
+              @stats[:errors] += 1
+            end
+          end
+        end
+      end
+
+      def stats
+        @lock.synchronize do
+          @stats.merge({
+            metrics_count: @metrics.size,
+            storage_size: @storage.size,
+            running: @running,
+            flush_interval: @flush_interval,
+            retention_period: @retention_period
+          })
+        end
+      end
+
+      def to_json
+        @lock.synchronize do
+          JSON.generate({
+            timestamp: Time.now.iso8601,
+            metrics: @metrics,
+            stats: @stats
+          })
+        end
+      end
+
+      # Render the current metrics using the Prometheus text exposition
+      # format. Labels are escaped and metric names are normalized so this
+      # output can be scraped without trusting caller-provided names.
+      def to_prometheus
+        @lock.synchronize do
+          lines = []
+          @metrics.each_value do |metric|
+            name = prometheus_name(metric[:name])
+            labels = prometheus_labels(metric[:labels])
+            case metric[:type]
+            when TYPE_COUNTER, TYPE_GAUGE
+              lines << "#{name}#{labels} #{metric[:value]}"
+            when TYPE_HISTOGRAM
+              values = metric[:values]
+              metric[:buckets].each do |bucket|
+                count = values.count { |value| value <= bucket }
+                lines << "#{name}_bucket#{prometheus_labels(metric[:labels].merge(le: bucket))} #{count}"
+              end
+              inf_label = "+Inf"
+              lines << "#{name}_bucket#{prometheus_labels(metric[:labels].merge(le: inf_label))} #{values.size}"
+              lines << "#{name}_count#{labels} #{values.size}"
+              lines << "#{name}_sum#{labels} #{values.sum}"
+            when TYPE_SUMMARY
+              sorted = metric[:values].sort
+              [0.5, 0.9, 0.99].each do |quantile|
+                index = [(sorted.size * quantile).ceil - 1, 0].max
+                value = sorted.empty? ? 0 : sorted[index]
+                lines << "#{name}#{prometheus_labels(metric[:labels].merge(quantile: quantile))} #{value}"
+              end
+              lines << "#{name}_count#{labels} #{sorted.size}"
+              lines << "#{name}_sum#{labels} #{sorted.sum}"
+            end
+          end
+          lines.join("\n") + (lines.empty? ? "" : "\n")
+        end
+      end
+
+      private
+
+      def metric_key(name, labels)
+        key = name.to_s.dup
+        labels.each do |k, v|
+          key << "_#{k}_#{v}"
+        end
+        key
+      end
+
+      def prometheus_name(name)
+        normalized = name.to_s.gsub(/[^a-zA-Z0-9_:]/, "_")
+        normalized = "rubydb_#{normalized}" unless normalized.start_with?("rubydb_")
+        normalized
+      end
+
+      def prometheus_labels(labels)
+        return "" if labels.empty?
+        encoded = labels.sort_by { |key, _| key.to_s }.map do |key, value|
+          escaped = value.to_s.gsub(/\\/, "\\\\").gsub('"', '\\"').gsub("\n", "\\n")
+          "#{key}=\"#{escaped}\""
+        end
+        "{#{encoded.join(',')}}"
+      end
+
+      def start_flush_thread
+        @running = true
+        @flush_thread = Thread.new do
+          while @running
+            sleep(@flush_interval)
+            begin
+              collect
+              flush
+            rescue => e
+              @stats[:errors] += 1
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rubydb/monitoring/performance.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+module RubyDB
+  module Monitoring
+    # Performance - Performance monitoring
+    class Performance
+      attr_reader :stats
+
+      def initialize(engine, config = {})
+        @engine = engine
+        @config = config
+        @statistics = Statistics.new
+        @metrics = {
+          query_times: [],
+          transaction_times: [],
+          connection_times: [],
+          wal_write_times: [],
+          checkpoint_times: [],
+          replication_lag: []
+        }
+        @query_counts = {}
+        @slow_queries = []
+        @max_slow_queries = config[:max_slow_queries] || 100
+        @slow_query_threshold = config[:slow_query_threshold] || 1000
+        @stats = {
+          total_queries: 0,
+          total_transactions: 0,
+          avg_query_time: 0,
+          slow_query_count: 0,
+          transaction_throughput: 0,
+          query_throughput: 0
+        }
+        @lock = Mutex.new
+        @start_time = Time.now
+      end
+
+      def record_query(query, time_ms)
+        @lock.synchronize do
+          @stats[:total_queries] += 1
+          @metrics[:query_times] << time_ms
+
+          # Track query type
+          type = extract_query_type(query)
+          @query_counts[type] ||= 0
+          @query_counts[type] += 1
+
+          # Record slow queries
+          if time_ms > @slow_query_threshold
+            @slow_queries << {
+              query: query,
+              time_ms: time_ms,
+              timestamp: Time.now.iso8601
+            }
+            @slow_queries = @slow_queries.last(@max_slow_queries)
+            @stats[:slow_query_count] += 1
+          end
+
+          update_stats
+        end
+      end
+
+      def record_transaction(time_ms)
+        @lock.synchronize do
+          @stats[:total_transactions] += 1
+          @metrics[:transaction_times] << time_ms
+          update_stats
+        end
+      end
+
+      def record_connection(time_ms)
+        @lock.synchronize do
+          @metrics[:connection_times] << time_ms
+          update_stats
+        end
+      end
+
+      def record_wal_write(time_ms)
+        @lock.synchronize do
+          @metrics[:wal_write_times] << time_ms
+          update_stats
+        end
+      end
+
+      def record_checkpoint(time_ms)
+        @lock.synchronize do
+          @metrics[:checkpoint_times] << time_ms
+          update_stats
+        end
+      end
+
+      def record_replication_lag(lag_ms)
+        @lock.synchronize do
+          @metrics[:replication_lag] << lag_ms
+          update_stats
+        end
+      end
+
+      def query_performance
+        @lock.synchronize do
+          times = @metrics[:query_times]
+          {
+            total: @stats[:total_queries],
+            avg: @statistics.mean(times),
+            median: @statistics.median(times),
+            p95: @statistics.percentile(times, 95),
+            p99: @statistics.percentile(times, 99),
+            min: @statistics.min(times),
+            max: @statistics.max(times)
+          }
+        end
+      end
+
+      def transaction_performance
+        @lock.synchronize do
+          times = @metrics[:transaction_times]
+          {
+            total: @stats[:total_transactions],
+            avg: @statistics.mean(times),
+            median: @statistics.median(times),
+            p95: @statistics.percentile(times, 95),
+            p99: @statistics.percentile(times, 99),
+            min: @statistics.min(times),
+            max: @statistics.max(times)
+          }
+        end
+      end
+
+      def slow_queries
+        @slow_queries.dup
+      end
+
+      def query_counts
+        @query_counts.dup
+      end
+
+      def throughput
+        @lock.synchronize do
+          elapsed = (Time.now - @start_time).to_f
+          {
+            queries_per_second: @stats[:total_queries] / elapsed,
+            transactions_per_second: @stats[:total_transactions] / elapsed
+          }
+        end
+      end
+
+      def stats
+        @lock.synchronize do
+          elapsed = (Time.now - @start_time).to_f
+          @stats.merge({
+            uptime_seconds: elapsed.round(2),
+            query_stats: query_performance,
+            transaction_stats: transaction_performance,
+            throughput: throughput,
+            slow_query_count: @slow_queries.size,
+            query_types: @query_counts
+          })
+        end
+      end
+
+      private
+
+      def extract_query_type(query)
+        query.strip.split.first&.upcase || "UNKNOWN"
+      end
+
+      def update_stats
+        times = @metrics[:query_times]
+        @stats[:avg_query_time] = @statistics.mean(times) if times.any?
+
+        elapsed = (Time.now - @start_time).to_f
+        @stats[:query_throughput] = @stats[:total_queries] / elapsed if elapsed > 0
+        @stats[:transaction_throughput] = @stats[:total_transactions] / elapsed if elapsed > 0
+      end
+    end
+  end
+end

data/lib/rubydb/monitoring/statistics.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+module RubyDB
+  module Monitoring
+    # Statistics - Statistical calculations
+    class Statistics
+      attr_reader :stats
+
+      def initialize
+        @stats = {
+          calculations: 0,
+          errors: 0
+        }
+        @lock = Mutex.new
+      end
+
+      def mean(values)
+        return 0 if values.empty?
+        values.sum / values.size.to_f
+      end
+
+      def median(values)
+        return 0 if values.empty?
+        sorted = values.sort
+        mid = sorted.size / 2
+        if sorted.size.even?
+          (sorted[mid - 1] + sorted[mid]) / 2.0
+        else
+          sorted[mid]
+        end
+      end
+
+      def mode(values)
+        return nil if values.empty?
+        freq = values.each_with_object(Hash.new(0)) { |v, h| h[v] += 1 }
+        max_freq = freq.values.max
+        freq.select { |_, f| f == max_freq }.keys
+      end
+
+      def variance(values)
+        return 0 if values.size < 2
+        m = mean(values)
+        values.sum { |v| (v - m) ** 2 } / (values.size - 1).to_f
+      end
+
+      def standard_deviation(values)
+        Math.sqrt(variance(values))
+      end
+
+      def percentile(values, p)
+        return 0 if values.empty?
+        sorted = values.sort
+        index = (p / 100.0) * (sorted.size - 1)
+        if index.to_i == index
+          sorted[index.to_i]
+        else
+          lower = sorted[index.floor]
+          upper = sorted[index.ceil]
+          lower + (upper - lower) * (index - index.floor)
+        end
+      end
+
+      def min(values)
+        values.min || 0
+      end
+
+      def max(values)
+        values.max || 0
+      end
+
+      def sum(values)
+        values.sum
+      end
+
+      def count(values)
+        values.size
+      end
+
+      def histogram(values, bins = 10)
+        return {} if values.empty?
+        min_val = min(values)
+        max_val = max(values)
+        range = max_val - min_val
+        bin_width = range / bins.to_f
+
+        histogram = {}
+        bins.times do |i|
+          low = min_val + i * bin_width
+          high = low + bin_width
+          key = "#{low.round(2)}-#{high.round(2)}"
+          histogram[key] = values.count { |v| v >= low && v < high }
+        end
+
+        histogram
+      end
+
+      def moving_average(values, window = 10)
+        return [] if values.empty?
+        result = []
+        values.each_with_index do |v, i|
+          start_idx = [0, i - window + 1].max
+          window_values = values[start_idx..i]
+          result << mean(window_values)
+        end
+        result
+      end
+
+      def exponential_moving_average(values, alpha = 0.1)
+        return [] if values.empty?
+        result = []
+        ema = values.first
+        values.each do |v|
+          ema = alpha * v + (1 - alpha) * ema
+          result << ema
+        end
+        result
+      end
+
+      def z_score(values, value)
+        return 0 if values.empty?
+        m = mean(values)
+        sd = standard_deviation(values)
+        return 0 if sd == 0
+        (value - m) / sd
+      end
+
+      def correlation(x_values, y_values)
+        return 0 if x_values.size != y_values.size || x_values.empty?
+        n = x_values.size
+        sum_x = x_values.sum
+        sum_y = y_values.sum
+        sum_xy = x_values.zip(y_values).sum { |x, y| x * y }
+        sum_x2 = x_values.sum { |x| x * x }
+        sum_y2 = y_values.sum { |y| y * y }
+
+        numerator = n * sum_xy - sum_x * sum_y
+        denominator = Math.sqrt((n * sum_x2 - sum_x * sum_x) * (n * sum_y2 - sum_y * sum_y))
+        return 0 if denominator == 0
+        numerator / denominator
+      end
+
+      def calculate_summary(values)
+        @stats[:calculations] += 1
+        {
+          count: count(values),
+          sum: sum(values),
+          mean: mean(values),
+          median: median(values),
+          min: min(values),
+          max: max(values),
+          variance: variance(values),
+          std_dev: standard_deviation(values),
+          p25: percentile(values, 25),
+          p50: percentile(values, 50),
+          p75: percentile(values, 75),
+          p90: percentile(values, 90),
+          p95: percentile(values, 95),
+          p99: percentile(values, 99)
+        }
+      rescue => e
+        @stats[:errors] += 1
+        {}
+      end
+
+      def stats
+        @stats
+      end
+    end
+  end
+end