circleci-tools 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 65da82885c98d74e328659ed86b6b2333781926b224d5d291339a982c35f1031
+  data.tar.gz: 04ba44f30782f8934656efe8822072d2bcaceebaf876a9f66ae62f5fb3cd28bf
+SHA512:
+  metadata.gz: bc5a56488a87728059c683603ca462e1243ad9e40eab1bd5bd986a927fa37e66383ccd0011255d20a82b880abd9d4047677f9ef1668a377a0e6fe737aabfe9bf
+  data.tar.gz: 9e6e4a915b075d0622b897935add9be6986a60c293bb10311eea82a3402076415b87ec5a623480a2ed71001976fb3d98ce98c8c8c4752b8b738fa9bc72e5f812

data/README.md

@@ -0,0 +1,61 @@
+# CircleCI Tools
+
+CircleCI Tools is a collection of utilities designed to enhance and streamline your CircleCI workflows. This CLI provides various commands to evaluate concurrency requirements, aggregate data, upload metrics, and generate usage reports.
+
+## Installation
+
+To set up the project, follow these steps:
+
+1. Clone the repository:
+   ```bash
+   git clone https://github.com/sofatutor/circleci-tools.git
+   cd circleci-tools
+   ```
+
+2. Install the dependencies:
+   ```bash
+   bundle install
+   ```
+
+## Usage
+
+The CLI provides the following commands:
+
+- **evaluate**: Evaluate concurrency requirements for self-hosted runners.
+  ```bash
+  bin/circleci-metrics evaluate --org=ORG_NAME --project=PROJECT_NAME
+  ```
+
+- **aggregate**: Aggregate data from an existing jobs JSON file.
+  ```bash
+  bin/circleci-metrics aggregate --jobs_json=JOBS_JSON_PATH
+  ```
+
+- **upload**: Store aggregated CSV data into SQLite database for analysis.
+  ```bash
+  bin/circleci-metrics upload --csv_file_path=CSV_FILE_PATH
+  ```
+
+- **usage_report**: Create usage export job, download CSV, and print file references.
+  ```bash
+  bin/circleci-metrics usage_report --org_id=ORG_ID
+  ```
+
+- **upload_metrics**: Upload CloudWatch metrics from CSV file.
+  ```bash
+  bin/circleci-metrics upload_metrics --csv_file_path=CSV_FILE_PATH
+  ```
+
+## Contributing
+
+We welcome contributions to enhance the functionality of CircleCI Tools. Please follow these steps to contribute:
+
+1. Fork the repository.
+2. Create a new branch for your feature or bugfix.
+3. Commit your changes with clear commit messages.
+4. Push your changes to your fork.
+5. Open a pull request with a detailed description of your changes.
+
+## License
+
+This project is licensed under the MIT License.

data/bin/circleci-metrics

@@ -0,0 +1,281 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.expand_path('../lib/circleci-tools', __dir__))
+
+require 'thor'
+require 'tty-prompt'
+require 'json'
+require 'csv'
+require 'date'
+require 'active_support/all'
+
+require 'api_service'
+require 'job_analyzer'
+require 'runner_calculator'
+require 'data_aggregator'
+require 'log_uploader'
+require 'usage_report_service'
+require 'cloudwatch_metrics_service'
+require 's3_upload_service'
+
+module CircleciMetrics
+  class CLI < Thor
+    desc "evaluate", "Evaluate concurrency requirements for self-hosted runners"
+
+    method_option :org, aliases: '-o', type: :string, desc: 'VCS org'
+    method_option :project, aliases: '-p', type: :string, desc: 'Project name'
+    method_option :days, aliases: '-d', type: :numeric, desc: 'Number of days to look back for pipelines', default: 30
+    method_option :pipelines_json, type: :string, desc: 'Path to JSON file with pre-fetched pipelines'
+    method_option :jobs_json, aliases: '-jjson', type: :string, desc: 'Path to JSON file with pre-fetched jobs'
+
+    def evaluate
+      prompt = TTY::Prompt.new
+
+      org = fetch_param(:org, 'CIRCLECI_ORG', prompt, "Enter your VCS org:")
+      project = fetch_param(:project, 'CIRCLECI_PROJECT', prompt, "Enter your project name:")
+      days = options[:days]
+
+      api_token = fetch_api_token(prompt)
+
+      circleci_service = CircleciTools::ApiService.new(api_token:, org:, project:)
+      job_analyzer = CircleciTools::JobAnalyzer.new
+      runner_calculator = CircleciTools::RunnerCalculator.new
+
+      pipelines = load_or_fetch_pipelines(circleci_service, org, project, days)
+      puts "Fetched pipelines: #{pipelines.size}"
+
+      return if pipelines.empty?
+
+      all_jobs = load_or_fetch_jobs(circleci_service, pipelines)
+      return if all_jobs.empty?
+
+      puts "Calculating peak RAM usage..."
+      peak_ram = job_analyzer.calculate_peak_ram(jobs: all_jobs)
+      puts "Peak concurrent RAM required: #{peak_ram} MB"
+
+      recommended_runners = runner_calculator.calculate_runners(peak_ram)
+      puts "Recommended number of runners (#{runner_calculator.runner_ram_gb} GB each): #{recommended_runners}"
+
+      aggregator = CircleciTools::DataAggregator.new(all_jobs)
+
+      aggregator.generate_csv
+    end
+
+    desc "upload", "Store aggregated CSV data into SQLite database for analysis"
+
+    method_option :csv_file_path, aliases: '-c', type: :string, required: true, desc: 'Path to the aggregated CSV file'
+    method_option :log_group_name, aliases: '-l', type: :string, default: '/CircleCi', desc: 'Log group name'
+    method_option :dry_run, type: :boolean, default: false, desc: 'Dry run mode'
+
+    def upload
+      csv_file_path = options[:csv_file_path]
+      log_group_name = options[:log_group_name]
+
+      importer = CircleciTools::LogUploader.new(log_group_name, dry_run: options[:dry_run])
+      importer.upload_file(csv_file_path)
+    end
+
+    desc "aggregate", "Aggregate data from an existing jobs JSON file"
+
+    method_option :jobs_json, aliases: '-j', type: :string, desc: 'Path to JSON file with jobs'
+
+    def aggregate
+      jobs_json_path = options[:jobs_json] || abort("Error: --jobs_json option is required")
+      jobs = JSON.parse(File.read(jobs_json_path))
+      aggregator = CircleciTools::DataAggregator.new(jobs)
+      aggregator.generate_csv
+    end
+
+    desc "usage_report", "Create usage export job, download CSV, and print file references"
+    method_option :org_id, type: :string, desc: 'Organization ID'
+    method_option :shared_org_ids, type: :array, desc: 'Shared organization IDs'
+    method_option :dry_run, type: :boolean, default: false, desc: 'Dry run mode'
+    method_option :verbose, aliases: '-v', type: :boolean, default: false, desc: 'Enable verbose logging'
+    method_option :days_ago, type: :numeric, default: 0, desc: 'Number of days to look back from now'
+    method_option :months_ago, type: :numeric, default: nil, desc: 'Number of months to look back from now'
+    method_option :usage_export_job_id, aliases: 'j', type: :string, desc: 'Existing usage export job ID'
+    method_option :upload, type: :boolean, default: false, desc: 'Upload the usage report to CloudWatch'
+    method_option :s3_bucket, type: :string, desc: 'S3 bucket name for uploading the usage report'
+    def usage_report
+      prompt = TTY::Prompt.new
+      org_id = options[:org_id] || ENV.fetch('CIRCLECI_ORG_ID', '4f925b2b-47d2-4775-8200-af0e5e2d9806')
+
+      shared_org_ids = options[:shared_org_ids] || []
+      api_token = fetch_api_token(prompt) # ...call existing helper or use logic shown in evaluate method
+      log_level = options[:verbose] ? Logger::DEBUG : Logger::INFO
+      logger = Logger.new(STDOUT)
+      logger.level = log_level
+
+      circleci_service = CircleciTools::ApiService.new(api_token:, org: 'N/A', project: 'N/A', logger:)
+      usage_report_service = CircleciTools::UsageReportService.new(
+        circleci_service,
+        org_id,
+        shared_org_ids,
+        600,
+        logger: logger,
+        log_level: log_level
+      )
+
+      usage_export_job_id = options[:usage_export_job_id]
+      if usage_export_job_id
+        downloaded_files = usage_report_service.call(usage_export_job_id:)
+      else
+        current_time = Time.now.utc
+        if options[:months_ago]
+          months_ago = options[:months_ago]
+          if months_ago.zero?
+            start_time = current_time.beginning_of_month
+            end_time = current_time
+          else
+            start_time = (current_time - months_ago.months).beginning_of_month
+            end_time = (current_time - months_ago.months).end_of_month
+          end
+        elsif options[:days_ago]
+          days_ago = options[:days_ago]
+          if days_ago.zero?
+            start_time = current_time.beginning_of_day
+            end_time = current_time
+          else
+            start_time = (current_time - days_ago.days).beginning_of_day
+            end_time = (current_time - days_ago.days).end_of_day
+          end
+        else
+          raise "Either days_ago or months_ago must be specified"
+        end
+
+        begin
+          downloaded_files = usage_report_service.call(start_time:, end_time:)
+        rescue RuntimeError => e
+          puts "Error: #{e.message}"
+          exit(1)
+        end
+      end
+
+      csv_files = downloaded_files.select { |file| file.end_with?('.csv') }
+
+      if csv_files.size == 0
+        puts "No usage report available for the given time range."
+      elsif csv_files.size == 1
+        puts "Usage report file downloaded: #{csv_files.first}"
+      else
+        puts "Usage report files downloaded:"
+        csv_files.each { |f| puts "  - #{f}" }
+      end
+
+      if options[:s3_bucket]
+        s3_service = CircleciTools::S3UploadService.new(options[:s3_bucket], logger: logger)
+        csv_files.each do |file|
+          s3_key = File.basename(file)
+          s3_service.upload_file(file, s3_key)
+        end
+      end
+
+      if options[:upload]
+        if !options[:s3_bucket]
+          unless prompt.yes?("Warning: No S3 bucket specified. Events could be uploaded multiple times. Do you want to continue?")
+            puts "Operation aborted."
+            exit(1)
+          end
+        end
+
+        metrics_service = CircleciTools::CloudWatchMetricsService.new(
+          namespace: 'CircleCI',
+          dry_run: options[:dry_run],
+          s3_bucket: options[:s3_bucket]
+        )
+        csv_files.each do |file|
+          metrics_service.upload_metrics(file)
+        end
+      end
+    end
+
+    desc "upload_metrics", "Upload CloudWatch metrics from CSV file"
+    method_option :csv_file_path, aliases: '-c', type: :string, required: true, desc: 'Path to the CSV file'
+    method_option :namespace, aliases: '-n', type: :string, default: 'CircleCI', desc: 'CloudWatch namespace'
+    method_option :dry_run, type: :boolean, default: false, desc: 'Dry run mode'
+    def upload_metrics
+      csv_file_path = options[:csv_file_path]
+      namespace = options[:namespace]
+      dry_run = options[:dry_run]
+
+      metrics_service = CircleciTools::CloudWatchMetricsService.new(namespace: namespace, dry_run: dry_run)
+      metrics_service.upload_metrics(csv_file_path)
+    end
+
+    no_commands do
+      def load_or_fetch_pipelines(circleci_service, org, project, days)
+        if options[:pipelines_json]
+          puts "Loading pipelines from JSON file: #{options[:pipelines_json]}"
+          JSON.parse(File.read(options[:pipelines_json]))
+        else
+          puts "Fetching pipelines for project #{org}/#{project} that ran in the last #{days} days..."
+          pipelines = circleci_service.fetch_pipelines(days: days)
+          puts "Total pipelines fetched: #{pipelines.size}"
+
+          timestamp = Time.now.strftime('%Y%m%d%H%M%S')
+          filename = "tmp/pipelines_#{org}_#{project}_#{timestamp}.json"
+          File.write(filename, JSON.pretty_generate(pipelines))
+          puts "Pipelines exported to #{filename}"
+
+          pipelines
+        end
+      end
+
+      def load_or_fetch_jobs(circleci_service, pipelines)
+        if options[:jobs_json]
+          puts "Loading jobs from JSON file: #{options[:jobs_json]}"
+          JSON.parse(File.read(options[:jobs_json]))
+        else
+          puts "Fetching jobs for all pipelines..."
+          all_jobs = circleci_service.fetch_all_jobs(pipelines)
+          puts "Total jobs fetched: #{all_jobs.size}"
+
+          timestamp = Time.now.strftime('%Y%m%d%H%M%S')
+          filename = "tmp/jobs_#{timestamp}.json"
+          File.write(filename, JSON.pretty_generate(all_jobs))
+          puts "Jobs exported to #{filename}"
+
+          all_jobs
+        end
+      end
+
+      def fetch_param(option_key, env_var, prompt, message)
+        if options[option_key]
+          options[option_key]
+        else
+          ENV.fetch(env_var) do
+            if $stdin.tty?
+              prompt.ask(message) { |q| q.required(true) }
+            else
+              abort("Error: Environment variable #{env_var} is not set and no option provided.")
+            end
+          end
+        end
+      rescue KeyError
+        if $stdin.tty?
+          prompt.ask(message) { |q| q.required(true) }
+        else
+          abort("Error: Environment variable #{env_var} is not set and no option provided.")
+        end
+      end
+
+      def fetch_api_token(prompt)
+        ENV.fetch('CIRCLECI_API_TOKEN') do
+          if $stdin.tty?
+            prompt.mask("Enter your CircleCI API Token:") { |q| q.required(true) }
+          else
+            abort("Error: Environment variable CIRCLECI_API_TOKEN is not set and no option provided.")
+          end
+        end
+      rescue KeyError
+        if $stdin.tty?
+          prompt.mask("Enter your CircleCI API Token:") { |q| q.required(true) }
+        else
+          abort("Error: Environment variable CIRCLECI_API_TOKEN is not set and no option provided.")
+        end
+      end
+    end
+  end
+end
+
+CircleciMetrics::CLI.start(ARGV)

data/lib/circleci-tools/api_service.rb

@@ -0,0 +1,177 @@
+# lib/circleci_concurrency_evaluator/circleci_service.rb
+
+require 'faraday'
+require 'json'
+require 'base64'
+require 'logger'
+require_relative 'retryable'
+
+module CircleciTools
+  class ApiService
+    include Retryable
+    BASE_URL = 'https://circleci.com'
+    MAX_THREADS = 10
+
+    def initialize(api_token:, org:, project:, vcs_type: 'gh', logger: Logger.new(STDOUT))
+      @api_token = api_token
+      @vcs_type = vcs_type
+      @org = org
+      @project = project
+      @logger = logger
+    end
+
+    def fetch_pipelines(days: nil)
+      pipelines = []
+      page_token = nil
+      cutoff_time = days ? Time.now - (days * 24 * 60 * 60) : nil
+
+      loop do
+        url = '/api/v2/pipeline'
+        params = {
+          'org-slug' => "#{@vcs_type}/#{@org}",
+          'page-token' => page_token,
+          'mine' => false
+        }
+
+        response = with_retries { connection.get(url, params.compact, headers) }
+        break unless response
+        raise "API Error: #{response.body}" unless response.status == 200
+
+        data = JSON.parse(response.body)
+
+        pipelines.concat(data['items'])
+
+        page_token = data['next_page_token']
+        break unless page_token
+
+        break if cutoff_time && data['items'].any? { |pipeline| Time.parse(pipeline['created_at']) < cutoff_time }
+      end
+
+      pipelines
+    end
+
+    def fetch_jobs(pipeline_id)
+      jobs = []
+      page_token = nil
+
+      loop do
+        url = "/api/v2/pipeline/#{pipeline_id}/workflow"
+        params = {}
+        params['page-token'] = page_token if page_token
+
+        response = with_retries { connection.get(url, params, headers) }
+        break unless response
+        raise "API Error: #{response.body}" unless response.status == 200
+
+        data = JSON.parse(response.body)
+        workflows = data['items']
+
+        threads = workflows.map do |workflow|
+          Thread.new do
+            workflow_jobs = fetch_workflow_jobs(workflow['id'])
+            jobs.concat(workflow_jobs)
+          end
+        end
+
+        threads.each(&:join)
+
+        page_token = data['next_page_token']
+        break unless page_token
+      end
+
+      jobs
+    end
+
+    def fetch_workflow_jobs(workflow_id)
+      url = "/api/v2/workflow/#{workflow_id}/job"
+
+      response = with_retries { connection.get(url, nil, headers) }
+      return [] unless response
+      raise "API Error: #{response.body}" unless response.status == 200
+
+      data = JSON.parse(response.body)
+      data['items']
+    end
+
+    def fetch_job_details(job)
+      url = "/api/v2/project/#{job['project_slug']}/job/#{job['job_number']}"
+
+      response = with_retries { connection.get(url, nil, headers) }
+      return nil unless response
+      raise "API Error: #{response.body}" unless response.status == 200
+
+      JSON.parse(response.body)
+    end
+
+    def fetch_all_jobs(pipelines)
+      all_jobs = []
+      semaphore = Mutex.new
+      threads = []
+
+      pipelines.each_with_index do |pipeline, index|
+        threads << Thread.new do
+          jobs = fetch_jobs(pipeline['id'])
+          jobs.each do |job|
+            next unless job['job_number']
+            next if job['status'] == 'not_run'
+
+            job_details = fetch_job_details(job)
+            next unless job_details
+            next unless job_details['duration']
+
+            semaphore.synchronize { all_jobs << job_details }
+          end
+          @logger.info("Fetched jobs for pipeline #{index + 1}/#{pipelines.size} (ID: #{pipeline['id']})")
+        end
+
+        if threads.size >= MAX_THREADS
+          threads.each(&:join)
+          threads.clear
+        end
+      end
+
+      threads.each(&:join)
+      all_jobs
+    end
+
+    def create_usage_export_job(org_id:, start_time:, end_time:, shared_org_ids: [])
+      url = "/api/v2/organizations/#{org_id}/usage_export_job"
+      body = { start: start_time, end: end_time, shared_org_ids: shared_org_ids }
+      response = with_retries(max_retries: 60) do
+        response = connection.post(url, body.to_json, headers.merge('Content-Type' => 'application/json'))
+        raise "API Error: #{response&.body}" unless response&.status == 201
+
+        response
+      end
+
+      return nil unless response
+
+      JSON.parse(response.body)
+    end
+
+    def get_usage_export_job(org_id:, usage_export_job_id:)
+      url = "/api/v2/organizations/#{org_id}/usage_export_job/#{usage_export_job_id}"
+      response = with_retries { connection.get(url, nil, headers) }
+      return nil unless response
+      raise "API Error: #{response.body}" unless response.status == 200
+
+      JSON.parse(response.body)
+    end
+
+    private
+
+    def connection
+      @connection ||= Faraday.new(url: BASE_URL) do |faraday|
+        faraday.request :url_encoded
+        faraday.adapter Faraday.default_adapter
+      end
+    end
+
+    def headers
+      {
+        'Circle-Token' => @api_token,
+        'Accept' => 'application/json'
+      }
+    end
+  end
+end