circleci-tools 0.1.0

data/lib/circleci-tools/cloudwatch_metrics_service.rb

@@ -0,0 +1,217 @@
+require 'aws-sdk-cloudwatch'
+require 'digest'
+require 'json'
+require 'set'
+require 'time'
+require 'tty-progressbar'
+
+module CircleciTools
+  class CloudWatchMetricsService
+    UPLOAD_BATCH_SIZE = 20
+    METRICS_DIGEST_FILENAME = 'cloud-watch-metrics-digests.txt'
+
+    def initialize(namespace: 'CircleCI', dry_run: false, logger: Logger.new(STDOUT), s3_bucket: nil)
+      @namespace = namespace
+      @dry_run = dry_run
+      @logger = logger
+      @cloudwatch = Aws::CloudWatch::Client.new
+      @s3_bucket = s3_bucket
+      @s3_client = Aws::S3::Client.new if @s3_bucket
+    end
+
+    def upload_metrics(file_path)
+      @logger.info("Uploading metrics from #{file_path} to CloudWatch...")
+      events = parse_csv(file_path)
+
+      if @dry_run
+        metrics = generate_metrics(events)
+        puts JSON.pretty_generate(metrics)
+      else
+        events.group_by { |event| event[:project_name] }.each do |project_name, project_events|
+          metrics = generate_metrics(project_events)
+          upload_to_cloudwatch(project_name, metrics)
+        end
+      end
+    end
+
+    private
+
+    def parse_csv(file_path)
+      events = []
+      two_weeks_ago = Time.now - (14 * 24 * 60 * 60)
+      CSV.foreach(file_path, headers: true) do |row|
+        next unless row['JOB_RUN_STARTED_AT'].to_i > 0 && row['JOB_RUN_STOPPED_AT'].to_i > 0
+
+        started_at = Time.parse(row['JOB_RUN_STARTED_AT'])
+        stopped_at = Time.parse(row['JOB_RUN_STOPPED_AT'])
+        next if stopped_at < two_weeks_ago
+
+        events << {
+          project_name: row['PROJECT_NAME'],
+          workflow_name: row['WORKFLOW_NAME'],
+          branch: row['VCS_BRANCH'],
+          job_name: row['JOB_NAME'],
+          job_status: row['JOB_BUILD_STATUS'],
+          started_at: started_at,
+          stopped_at: stopped_at,
+          run_time: (stopped_at - started_at).to_i,
+          compute_credits_used: row['COMPUTE_CREDITS'].to_i,
+          avg_ram: row['MEDIAN_RAM_UTILIZATION_PCT'].to_i,
+          max_ram: row['MAX_RAM_UTILIZATION_PCT'].to_i,
+          avg_cpu: row['MEDIAN_CPU_UTILIZATION_PCT'].to_i,
+          max_cpu: row['MAX_CPU_UTILIZATION_PCT'].to_i
+        }
+      end
+      events
+    end
+
+    def generate_metrics(events)
+      metrics = []
+      events.each do |event|
+        workflow_dimensions = [
+          { name: 'WorkflowName', value: event[:workflow_name] }
+        ]
+
+        branch_dimensions = [
+          { name: 'Branch', value: event[:branch] },
+          { name: 'JobName', value: event[:job_name] }
+        ]
+
+        truncated_timestamp = truncate_to_minute(event[:stopped_at])
+
+        metrics << {
+          metric_name: 'JobRunTime',
+          dimensions: branch_dimensions,
+          timestamp: truncated_timestamp,
+          value: event[:run_time],
+          unit: 'Seconds'
+        } if event[:run_time] > 0
+        metrics << {
+          metric_name: 'AverageRAMUtilization',
+          dimensions: branch_dimensions,
+          timestamp: truncated_timestamp,
+          value: event[:avg_ram],
+          unit: 'Percent'
+        } if event[:avg_ram] > 0
+        metrics << {
+          metric_name: 'MaxRAMUtilization',
+          dimensions: branch_dimensions,
+          timestamp: truncated_timestamp,
+          value: event[:max_ram],
+          unit: 'Percent'
+        } if event[:max_ram] > 0
+        metrics << {
+          metric_name: 'AverageCPUUtilization',
+          dimensions: branch_dimensions,
+          timestamp: truncated_timestamp,
+          value: event[:avg_cpu],
+          unit: 'Percent'
+        } if event[:avg_cpu] > 0
+        metrics << {
+          metric_name: 'MaxCPUUtilization',
+          dimensions: branch_dimensions,
+          timestamp: truncated_timestamp,
+          value: event[:max_cpu],
+          unit: 'Percent'
+        } if event[:max_cpu] > 0
+        metrics << {
+          metric_name: 'JobSucceeded',
+          dimensions: branch_dimensions,
+          timestamp: truncated_timestamp,
+          value: 1,
+          unit: 'Count'
+        } if event[:job_status] == 'success'
+        metrics << {
+          metric_name: 'JobFailed',
+          dimensions: branch_dimensions,
+          timestamp: truncated_timestamp,
+          value: 1,
+          unit: 'Count'
+        } if event[:job_status] == 'failed'
+        metrics << {
+          metric_name: 'ComputeCreditsUsed',
+          dimensions: workflow_dimensions,
+          timestamp: truncated_timestamp,
+          value: event[:compute_credits_used],
+          unit: 'Count'
+        } if event[:compute_credits_used] > 0
+      end
+      metrics
+    end
+
+    def truncate_to_minute(time)
+      Time.at(time.to_i - time.sec)
+    end
+
+    def upload_to_cloudwatch(project_name, metrics)
+      bar = TTY::ProgressBar.new("Uploading [:bar] :percent :elapsed", total: metrics.size)
+
+      existing_digests = load_existing_digests
+      new_metrics = []
+      new_digests = []
+
+      metrics.each do |metric|
+        digest = Digest::MD5.hexdigest(metric.to_s)
+        next if existing_digests.include?(digest)
+
+        new_metrics << metric
+        new_digests << digest
+      end
+
+      new_metrics.each_slice(UPLOAD_BATCH_SIZE) do |metric_batch|
+        begin
+          @cloudwatch.put_metric_data(
+            namespace: "#{@namespace}/#{project_name}",
+            metric_data: metric_batch
+          )
+          bar.advance(metric_batch.size)
+        rescue Aws::CloudWatch::Errors::ServiceError => e
+          @logger.error("Failed to upload metrics: #{e.message}")
+        end
+      end
+
+      store_new_digests(new_digests)
+
+      @logger.info("Uploaded #{new_metrics.size} metrics to CloudWatch for project #{project_name}.")
+    end
+
+    private
+
+    def load_existing_digests
+      if @s3_bucket
+        begin
+          resp = @s3_client.get_object(bucket: @s3_bucket, key: "#{@namespace.downcase}/#{METRICS_DIGEST_FILENAME}")
+          Set.new(resp.body.read.split("\n"))
+        rescue Aws::S3::Errors::NoSuchKey
+          Set.new
+        end
+      else
+        digest_file = File.join('tmp', METRICS_DIGEST_FILENAME)
+        if File.exist?(digest_file)
+          Set.new(digest_file).map(&:chomp)
+        else
+          Set.new
+        end
+      end
+    end
+
+    def store_new_digests(new_digests)
+      return if new_digests.empty?
+
+      if @s3_bucket
+        old_digests = load_existing_digests
+        merged_digests = (old_digests + new_digests).to_a.uniq
+        rotated_digests = merged_digests.last(100_000).join("\n")
+        @s3_client.put_object(bucket: @s3_bucket, key: "#{@namespace.downcase}/#{METRICS_DIGEST_FILENAME}", body: rotated_digests)
+      else
+        digest_file = File.join('tmp', METRICS_DIGEST_FILENAME)
+        old_digests = File.exist?(digest_file) ? File.readlines(digest_file).map(&:chomp) : []
+        merged_digests = (old_digests + new_digests).uniq
+        rotated_digests = merged_digests.last(100_000)
+        File.open(digest_file, 'w') do |file|
+          rotated_digests.each { |digest| file.puts digest }
+        end
+      end
+    end
+  end
+end

data/lib/circleci-tools/data_aggregator.rb

@@ -0,0 +1,57 @@
+module CircleciTools
+  class DataAggregator
+    CREDIT_COST = 0.0006
+
+    RESOURCE_CLASS_MAP = {
+      "small" => { cpus: 1, ram: 2 },
+      "medium" => { cpus: 2, ram: 4 },
+      "medium+" => { cpus: 3, ram: 6 },
+      "large" => { cpus: 4, ram: 8 }
+    }
+
+    def initialize(jobs)
+      @jobs = jobs
+    end
+
+    def generate_csv
+      csv_file_path = 'tmp/jobs_aggregated.csv'
+
+      CSV.open(csv_file_path, 'w') do |csv|
+        csv << [
+          'job_number', 'duration (ms)', 'duration_minutes', 'total_duration_minutes', 'queued_at',
+          'started_at', 'stopped_at', 'status', 'parallelism', 'resource_class', 'name',
+          'CPUs', 'RAM', 'total_ram', 'total_cpus', 'total_credits', 'total_costs'
+        ]
+
+        @jobs.each do |job|
+          duration = job['duration'] || 0
+          duration_minutes = duration / 1000.0 / 60.0
+          parallelism = job['parallelism'] || 1
+          total_duration_minutes = duration_minutes * parallelism
+
+          resource_class = job['executor']['resource_class']
+          next unless resource_class
+
+          mapped_class = RESOURCE_CLASS_MAP[resource_class]
+          next unless mapped_class
+
+          cpus = mapped_class[:cpus] || 1
+          ram = mapped_class[:ram] || 1
+
+          total_ram = parallelism * ram
+          total_cpus = parallelism * cpus
+          total_credits = total_cpus * duration_minutes * 5
+          total_costs = (total_credits * CREDIT_COST * parallelism).round(2)
+
+          csv << [
+            job['number'], duration, duration_minutes, total_duration_minutes, job['queued_at'],
+            job['started_at'], job['stopped_at'], job['status'], parallelism, resource_class,
+            job['name'], cpus, ram, total_ram, total_cpus, total_credits, total_costs
+          ]
+        end
+      end
+
+      puts "CSV file created at #{csv_file_path}"
+    end
+  end
+end

data/lib/circleci-tools/job_analyzer.rb

@@ -0,0 +1,58 @@
+require 'time'
+
+module CircleciTools
+  class JobAnalyzer
+    RESOURCE_RAM = {
+      'small' => 2048,   # in MB
+      'medium' => 4096,
+      'medium+' => 6144,
+      'large' => 8192,
+      # Add other classes if necessary
+    }.freeze
+
+    def calculate_peak_ram(jobs:)
+      events = []
+
+      jobs.each do |job|
+        next unless job['started_at'] && job['stopped_at']
+
+        start = parse_time(job['started_at'])
+        end_time = parse_time(job['stopped_at'])
+        ram = get_ram_claim(job)
+
+        events << { time: start, type: 'start', ram: ram }
+        events << { time: end_time, type: 'end', ram: ram }
+      end
+
+      # Sort events by time; 'end' before 'start' if times are equal
+      events.sort_by! { |event| [event[:time], event[:type] == 'end' ? 0 : 1] }
+
+      current_ram = 0
+      peak_ram = 0
+
+      events.each do |event|
+        if event[:type] == 'start'
+          current_ram += event[:ram]
+          peak_ram = [peak_ram, current_ram].max
+        else
+          current_ram -= event[:ram]
+        end
+      end
+
+      peak_ram
+    end
+
+    private
+
+    def get_ram_claim(job)
+      resource_class = job['executor']['resource_class'] || 'medium'  # Default to 'medium' if not specified
+      RESOURCE_RAM[resource_class] || 4096  # Default to 4096 MB if class not found
+    end
+
+    def parse_time(time_str)
+      Time.parse(time_str)
+    rescue
+      nil
+    end
+  end
+end

data/lib/circleci-tools/log_uploader.rb

@@ -0,0 +1,274 @@
+require 'aws-sdk-cloudwatchlogs'
+require 'time'
+require 'csv'
+require 'json'
+require_relative 'retryable'
+require 'tty-prompt'
+require 'date'
+
+module CircleciTools
+  class LogUploader
+    MAX_THREADS = 5
+
+    include Retryable
+
+    def initialize(log_group_name, dry_run: false)
+      @log_group_name = log_group_name
+      @dry_run = dry_run
+      @client = Aws::CloudWatchLogs::Client.new
+
+      ensure_log_group_exists
+    end
+
+    def upload_file(file_path)
+      events = generate_events(file_path)
+      events.sort_by! { |event| event[:timestamp] }
+
+      grouped_events = group_events_by_date(events)
+
+      if @dry_run
+        handle_dry_run(grouped_events)
+      else
+        upload_grouped_events(grouped_events, file_path)
+      end
+    end
+
+    private
+
+    def generate_events(file_path)
+      events = []
+      interval = 10 # seconds
+
+      CSV.foreach(file_path, headers: true) do |row|
+        queued_at = Time.parse(row['queued_at'])
+        started_at = Time.parse(row['started_at']) rescue nil
+        stopped_at = Time.parse(row['stopped_at']) rescue nil
+
+        next unless queued_at && started_at && stopped_at
+
+        # Initialize current_time to the next 10-second interval after queued_at
+        current_time = align_time_to_next_interval(queued_at, interval)
+        end_time = stopped_at
+
+        until current_time >= end_time
+          state = determine_state(current_time, started_at, stopped_at)
+
+          log_data = {
+            job_number: row['job_number'],
+            state: state,
+            name: row['name'],
+            total_ram: row['total_ram'],
+            total_cpus: row['total_cpus'],
+          }
+
+          events << {
+            timestamp: current_time.to_i * 1000,
+            message: log_data.to_json
+          }
+
+          current_time += interval
+        end
+
+        completed_time = align_time_to_next_interval(stopped_at, interval)
+        log_data = {
+          job_number: row['job_number'],
+          state: 'completed',
+          name: row['name'],
+          total_ram: row['total_ram'],
+          total_cpus: row['total_cpus'],
+        }
+
+        events << {
+          timestamp: completed_time.to_i * 1000,
+          message: log_data.to_json
+        }
+      end
+
+      events
+    rescue => e
+      puts "Error generating events: #{e.message}"
+      []
+    end
+
+    # Extracted Method: Group Events by Date
+    def group_events_by_date(events)
+      events.group_by do |event|
+        Time.at(event[:timestamp] / 1000).utc.strftime('%Y-%m-%d')
+      end
+    end
+
+    # Extracted Method: Handle Dry Run for Grouped Events
+    def handle_dry_run(grouped_events)
+      # Determine the range of dates
+      dates = grouped_events.keys.sort
+      from_date = dates.first
+      to_date = dates.last
+      output_file = "tmp/circleci-job_events-#{from_date}_to_#{to_date}.csv"
+
+      CSV.open(output_file, 'w') do |csv|
+        # Define CSV headers based on log_data fields
+        csv << ['timestamp', 'job_number', 'state', 'name', 'total_ram', 'total_cpus']
+
+        grouped_events.each do |date, events|
+          events.each do |event|
+            log_data = JSON.parse(event[:message])
+            timestamp = Time.at(event[:timestamp] / 1000).utc.iso8601
+            csv << [
+              timestamp,
+              log_data['job_number'],
+              log_data['state'],
+              log_data['name'],
+              log_data['total_ram'],
+              log_data['total_cpus']
+            ]
+          end
+        end
+      end
+      puts "Dry run enabled: All events stored in #{output_file}"
+    rescue => e
+      puts "Error during dry run: #{e.message}"
+    end
+
+    # Extracted Method: Upload Grouped Events to CloudWatch
+    def upload_grouped_events(grouped_events, file_path)
+      grouped_events.each do |date, events|
+        log_stream_name = "jobs-#{date}"
+        ensure_log_stream_exists(log_stream_name)
+
+        upload_events_to_stream(events, log_stream_name, date)
+      end
+      puts "Uploaded #{file_path} to CloudWatch log streams in log group #{@log_group_name}"
+    rescue => e
+      puts "Error uploading events: #{e.message}"
+    end
+
+    def determine_state(current_time, started_at, stopped_at)
+      if current_time < started_at
+        'running'
+      elsif current_time >= started_at && current_time < stopped_at
+        'running'
+      elsif current_time >= stopped_at
+        'completed'
+      else
+        'unknown'
+      end
+    end
+
+    # Aligns a given time to the next 10-second interval and returns a Time object
+    def align_time_to_next_interval(time, interval)
+      aligned_seconds = (time.to_f / interval).ceil * interval
+      Time.at(aligned_seconds).utc
+    end
+
+    # Extracted Method: Ensure Log Group Exists
+    def ensure_log_group_exists
+      log_group = @client.describe_log_groups(log_group_name_prefix: @log_group_name).log_groups.find { |lg| lg.log_group_name == @log_group_name }
+
+      unless log_group
+        if $stdin.tty?
+          prompt = TTY::Prompt.new
+          create = prompt.yes?("Log group '#{@log_group_name}' does not exist. Would you like to create it?")
+
+          if create
+            @client.create_log_group(log_group_name: @log_group_name)
+            puts "Created log group '#{@log_group_name}'."
+          else
+            abort("Log group '#{@log_group_name}' does not exist. Exiting.")
+          end
+        else
+          abort("Log group '#{@log_group_name}' does not exist and no interactive prompt available. Exiting.")
+        end
+      end
+    end
+
+    # Modify ensure_log_stream_exists to handle deletion of existing log streams with prompt
+    def ensure_log_stream_exists(log_stream_name)
+      log_stream = @client.describe_log_streams(
+        log_group_name: @log_group_name,
+        log_stream_name_prefix: log_stream_name
+      ).log_streams.find { |stream| stream.log_stream_name == log_stream_name }
+
+      if log_stream
+        if $stdin.tty?
+          prompt = TTY::Prompt.new
+          delete = prompt.yes?("Log stream '#{log_stream_name}' already exists in log group '#{@log_group_name}'. Would you like to delete it and create a new one?")
+
+          if delete
+            @client.delete_log_stream(log_group_name: @log_group_name, log_stream_name: log_stream_name)
+            puts "Deleted existing log stream '#{log_stream_name}'."
+            @client.create_log_stream(log_group_name: @log_group_name, log_stream_name: log_stream_name)
+            puts "Created new log stream '#{log_stream_name}' in log group '#{@log_group_name}'."
+          else
+            abort("Log stream '#{log_stream_name}' already exists. Exiting.")
+          end
+        else
+          abort("Log stream '#{log_stream_name}' already exists in log group '#{@log_group_name}' and no interactive prompt available. Exiting.")
+        end
+      else
+        @client.create_log_stream(log_group_name: @log_group_name, log_stream_name: log_stream_name)
+        puts "Created log stream '#{log_stream_name}' in log group '#{@log_group_name}'."
+      end
+    end
+
+    # Extracted Method: Upload Events to a Specific Log Stream
+    def upload_events_to_stream(events, log_stream_name, date)
+      batches = events.each_slice(1_000).to_a
+      threads = []
+
+      batches.each_with_index do |events_batch, index|
+        threads << Thread.new do
+          with_retries do
+            params = {
+              log_events: events_batch,
+              log_group_name: @log_group_name,
+              log_stream_name: log_stream_name
+            }
+
+            # Get the sequence token for the log stream
+            response = @client.describe_log_streams(
+              log_group_name: @log_group_name,
+              log_stream_name_prefix: log_stream_name
+            )
+            log_stream = response.log_streams.find { |stream| stream.log_stream_name == log_stream_name }
+            if log_stream && log_stream.upload_sequence_token
+              params[:sequence_token] = log_stream.upload_sequence_token
+            end
+
+            # Upload the log events
+            @client.put_log_events(params)
+          end
+          puts "Uploaded batch #{index + 1}/#{batches.size} for date #{date} to CloudWatch."
+        end
+
+        if threads.size >= MAX_THREADS
+          threads.each(&:join)
+          threads.clear
+        end
+      end
+
+      threads.each(&:join)
+    end
+
+    def send_log(log_group_name, log_stream_name, message)
+      params = {
+        log_events: [{
+          timestamp: (Time.now.to_f * 1000).to_i,
+          message: message
+        }],
+        log_group_name: log_group_name,
+        log_stream_name: log_stream_name
+      }
+
+      response = @client.describe_log_streams(
+        log_group_name: log_group_name,
+        log_stream_name_prefix: log_stream_name
+      )
+      log_stream = response.log_streams.find { |stream| stream.log_stream_name == log_stream_name }
+      if log_stream && log_stream.upload_sequence_token
+        params[:sequence_token] = log_stream.upload_sequence_token
+      end
+
+      @client.put_log_events(params)
+    end
+  end
+end

data/lib/circleci-tools/retryable.rb

@@ -0,0 +1,29 @@
+module CircleciTools
+  module Retryable
+    MAX_RETRIES = 5
+    BACKOFF_FACTOR = 0.5
+    MAX_BACKOFF_TIME = 60
+
+    def with_retries(max_retries: MAX_RETRIES)
+      retries = 0
+      begin
+        yield
+      rescue => e
+        if retries < max_retries
+          retries += 1
+          backoff_time = [BACKOFF_FACTOR * (2 ** retries), MAX_BACKOFF_TIME].min.floor
+          retry_logger.info "Retry ##{retries} after #{backoff_time} seconds"
+          retry_logger.debug "Thread #{Thread.current.object_id}: Error: #{e.message}"
+          sleep backoff_time
+          retry
+        else
+          retry_logger.warn "Thread #{Thread.current.object_id}: Error: #{e.message}"
+        end
+      end
+    end
+
+    def retry_logger
+      @logger ||= Logger.new(STDOUT)
+    end
+  end
+end

data/lib/circleci-tools/runner_calculator.rb

@@ -0,0 +1,14 @@
+module CircleciTools
+  class RunnerCalculator
+    attr_reader :runner_ram_gb
+
+    def initialize(runner_ram_gb = 8)
+      @runner_ram_gb = runner_ram_gb
+      @runner_ram_mb = @runner_ram_gb * 1024  # Convert GB to MB
+    end
+
+    def calculate_runners(peak_ram_mb)
+      (peak_ram_mb.to_f / @runner_ram_mb).ceil
+    end
+  end
+end

data/lib/circleci-tools/s3_upload_service.rb

@@ -0,0 +1,20 @@
+require 'aws-sdk-s3'
+require 'logger'
+
+module CircleciTools
+  class S3UploadService
+    def initialize(bucket_name, logger: Logger.new(STDOUT))
+      @bucket_name = bucket_name
+      @logger = logger
+      @s3_client = Aws::S3::Client.new
+    end
+
+    def upload_file(file_path, s3_key)
+      @logger.info("Uploading #{file_path} to S3 bucket #{@bucket_name} with key #{s3_key}...")
+      @s3_client.put_object(bucket: @bucket_name, key: "circleci/#{s3_key}", body: File.read(file_path))
+      @logger.info("Uploaded #{file_path} to S3 bucket #{@bucket_name} with key #{s3_key}.")
+    rescue Aws::S3::Errors::ServiceError => e
+      @logger.error("Failed to upload #{file_path} to S3: #{e.message}")
+    end
+  end
+end