aptible-cli 0.19.4 → 0.19.7

data/lib/aptible/cli/helpers/s3_log_helpers.rb

@@ -0,0 +1,214 @@
+require 'aws-sdk'
+require 'pathname'
+
+module Aptible
+  module CLI
+    module Helpers
+      module S3LogHelpers
+        include Helpers::DateHelpers
+
+        def ensure_aws_creds
+          cred_errors = []
+          unless ENV['AWS_ACCESS_KEY_ID']
+            cred_errors << 'Missing environment variable: AWS_ACCESS_KEY_ID'
+          end
+          unless ENV['AWS_SECRET_ACCESS_KEY']
+            cred_errors << 'Missing environment variable: AWS_SECRET_ACCESS_KEY'
+          end
+          raise Thor::Error, cred_errors.join(' ') if cred_errors.any?
+        end
+
+        def validate_log_search_options(options = {})
+          id_options = [
+            options[:app_id],
+            options[:database_id],
+            options[:endpoint_id],
+            options[:container_id]
+          ]
+          date_options = [options[:start_date], options[:end_date]]
+          unless options[:string_matches] || id_options.any?
+            m = 'You must specify an option to identify the logs to download,' \
+                ' either: --string-matches, --app-id, --database-id,' \
+                ' --endpoint-id, or --container-id'
+            raise Thor::Error, m
+          end
+
+          m = 'You cannot pass --app-id, --database-id, --endpoint-id, or ' \
+              '--container-id when using --string-matches.'
+          raise Thor::Error, m if options[:string_matches] && id_options.any?
+
+          m = 'You must specify only one of ' \
+              '--app-id, --database-id, --endpoint-id or --container-id'
+          raise Thor::Error, m if id_options.any? && !id_options.one?
+
+          m = 'The options --start-date/--end-date cannot be used when ' \
+              'searching by string'
+          raise Thor::Error, m if options[:string_matches] && date_options.any?
+
+          m = 'You must pass both --start-date and --end-date'
+          raise Thor::Error, m if date_options.any? && !date_options.all?
+
+          if options[:container_id] && options[:container_id].length < 12
+            m = 'You must specify at least the first 12 characters of the ' \
+                'container ID'
+            raise Thor::Error, m
+          end
+
+          if options[:download_location] && !options[:decryption_keys]
+            m = 'You must provide decryption keys with the --decryption-keys' \
+                'option in order to download files.'
+            raise Thor::Error, m
+          end
+        end
+
+        def info_from_path(file)
+          properties = {}
+
+          properties[:stack], _, properties[:schema],
+            properties[:shasum], type_id, *remainder = file.split('/')
+
+          properties[:id] = type_id.split('-').last.to_i
+          properties[:type] = type_id.split('-').first
+
+          case properties[:schema]
+          when 'v2'
+            # Eliminate the extensions
+            split_by_dot = remainder.pop.split('.') - %w(log bck gz)
+            properties[:container_id] = split_by_dot.first.delete!('-json')
+            properties[:uploaded_at] = utc_datetime(split_by_dot.last)
+          when 'v3'
+            case properties[:type]
+            when 'apps'
+              properties[:service_id] = remainder.first.split('-').last.to_i
+              file_name = remainder.second
+            else
+              file_name = remainder.first
+            end
+            # The file name may have differing number of elements due to
+            # docker file log rotation. So we eliminate some useless items
+            # and then work from the beginning or end of the remaining to find
+            # known elements, ignoring any .1 .2 (or none at all) extension
+            # found in the middle of the file name. EG:
+            # ['container_id', 'start_time', 'end_time']
+            # or
+            # ['container_id', '.1', 'start_time', 'end_time']]
+            split_by_dot = file_name.split('.') - %w(log gz archived)
+            properties[:container_id] = split_by_dot.first.delete!('-json')
+            properties[:start_time] = utc_datetime(split_by_dot[-2])
+            properties[:end_time] = utc_datetime(split_by_dot[-1])
+          else
+            m = "Cannot determine aptible log naming schema from #{file}"
+            raise Thor::Error, m
+          end
+          properties
+        end
+
+        def decrypt_and_translate_s3_file(file, enc_key, region, bucket, path)
+          # AWS warns us about using the legacy encryption schema
+          s3 = Kernel.silence_warnings do
+            Aws::S3::EncryptionV2::Client.new(
+              encryption_key: enc_key, region: region,
+              key_wrap_schema: :aes_gcm,
+              content_encryption_schema: :aes_gcm_no_padding,
+              security_profile: :v2_and_legacy
+            )
+          end
+
+          # Just write it to a file directly
+          location = File.join(path, file.split('/').drop(4).join('/'))
+          FileUtils.mkdir_p(File.dirname(location))
+          File.open(location, 'wb') do |f|
+            CLI.logger.info location
+            # Is this memory efficient?
+            s3.get_object(bucket: bucket, key: file, response_target: f)
+          end
+        end
+
+        def find_s3_files_by_string_match(region, bucket, stack, strings)
+          # This function just regex matches a provided string anywhwere
+          # in the s3 path
+          begin
+            stack_logs = s3_client(region).bucket(bucket)
+                                          .objects(prefix: stack)
+                                          .map(&:key)
+          rescue => error
+            raise Thor::Error, error.message
+          end
+          strings.each do |s|
+            stack_logs = stack_logs.select { |f| f =~ /#{s}/ }
+          end
+          stack_logs
+        end
+
+        def find_s3_files_by_attrs(region, bucket, stack,
+                                   attrs, time_range = nil)
+          # This function uses the known path schema to return files matching
+          # any provided criteria. EG:
+          # * attrs: { :type => 'app', :id => 123 }
+          # * attrs: { :container_id => 'deadbeef' }
+
+          begin
+            stack_logs = s3_client(region).bucket(bucket)
+                                          .objects(prefix: stack)
+                                          .map(&:key)
+          rescue => error
+            raise Thor::Error, error.message
+          end
+          attrs.each do |k, v|
+            stack_logs = stack_logs.select do |f|
+              if k == :container_id
+                # Match short container IDs
+                info_from_path(f)[k].start_with?(v)
+              else
+                info_from_path(f)[k] == v
+              end
+            end
+          end
+
+          if time_range
+            # select only logs within the time range
+            stack_logs = stack_logs.select do |f|
+              info = info_from_path(f)
+              first_log = info[:start_time]
+              last_log = info[:end_time]
+              if first_log.nil? || last_log.nil?
+                m = 'Cannot determine precise timestamps of file: ' \
+                    "#{f.split('/').drop(4).join('/')}"
+                CLI.logger.warn m
+                false
+              else
+                time_match?(time_range, first_log, last_log)
+              end
+            end
+          end
+
+          stack_logs
+        end
+
+        def time_match?(time_range, start_timestamp, end_timestamp)
+          return false if start_timestamp.nil? || end_timestamp.nil?
+          return false if time_range.last < start_timestamp
+          return false if time_range.first > end_timestamp
+          true
+        end
+
+        def encryption_key(filesum, possible_keys)
+          # The key can be determined from the sum
+          possible_keys.each do |k|
+            keysum = Digest::SHA256.hexdigest(Base64.strict_decode64(k))
+            next unless keysum == filesum
+            return Base64.strict_decode64(k)
+          end
+          m = "Did not find a matching key for shasum #{filesum}"
+          raise Thor::Error, m
+        end
+
+        def s3_client(region)
+          @s3_client ||= Kernel.silence_warnings do
+            Aws::S3::Resource.new(region: region)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aptible/cli/resource_formatter.rb

@@ -2,6 +2,8 @@ module Aptible
   module CLI
     module ResourceFormatter
       class << self
+        include Helpers::DateHelpers
+
         NO_NESTING = Object.new.freeze
 
         def inject_backup(node, backup, include_db: false)
@@ -50,8 +52,7 @@ module Aptible
             end
           end
 
-          if bu_operation && \
-             backup.manual && !backup.copied_from
+          if bu_operation && !backup.copied_from
             node.keyed_object('created_from_operation', 'id') do |n|
               inject_operation(n, bu_operation)
             end
@@ -236,6 +237,28 @@ module Aptible
           attach_account(node, account)
         end
 
+        def inject_maintenance(
+          node,
+          command_prefix,
+          maintenance_resource,
+          account
+        )
+          node.value('id', maintenance_resource.id)
+          raw_start, raw_end = maintenance_resource.maintenance_deadline
+          window_start = utc_string(raw_start)
+          window_end = utc_string(raw_end)
+          label = "#{maintenance_resource.handle} between #{window_start} "\
+                  "and #{window_end}"
+          restart_command = "#{command_prefix}"\
+                            " #{maintenance_resource.handle}"\
+                            " --environment #{account.handle}"
+          node.value('label', label)
+          node.value('handle', maintenance_resource.handle)
+          node.value('restart_command', restart_command)
+
+          attach_account(node, account)
+        end
+
         private
 
         def attach_account(node, account)

data/lib/aptible/cli/subcommands/deploy.rb

@@ -41,7 +41,7 @@ module Aptible
               git_ref = options[:git_commitish]
               if options[:git_detach]
                 if git_ref
-                  raise Thor::Error, 'The options --git-committish and ' \
+                  raise Thor::Error, 'The options --git-commitish and ' \
                                      '--git-detach are incompatible'
                 end
                 git_ref = NULL_SHA1

data/lib/aptible/cli/subcommands/logs.rb

@@ -1,4 +1,6 @@
+require 'aws-sdk'
 require 'shellwords'
+require 'time'
 
 module Aptible
   module CLI
@@ -8,6 +10,7 @@ module Aptible
           thor.class_eval do
             include Helpers::Operation
             include Helpers::AppOrDatabase
+            include Helpers::DateHelpers
 
             desc 'logs [--app APP | --database DATABASE]',
                  'Follows logs from a running app or database'
@@ -25,6 +28,148 @@ module Aptible
               ENV['ACCESS_TOKEN'] = fetch_token
               exit_with_ssh_portal(op, '-o', 'SendEnv=ACCESS_TOKEN', '-T')
             end
+
+            desc 'logs_from_archive --bucket NAME --region REGION ' \
+                 '--stack NAME [ --decryption-keys ONE [OR MORE] ] ' \
+                 '[ --download-location LOCATION ] ' \
+                 '[ [ --string-matches ONE [OR MORE] ] ' \
+                 '| [ --app-id ID | --database-id ID | --endpoint-id ID | ' \
+                 '--container-id ID ] ' \
+                 '[ --start-date YYYY-MM-DD --end-date YYYY-MM-DD ] ]',
+                 'Retrieves container logs from an S3 archive in your own ' \
+                 'AWS account. You must provide your AWS credentials via ' \
+                 'the environment variables AWS_ACCESS_KEY_ID and ' \
+                 'AWS_SECRET_ACCESS_KEY'
+
+            # Required to retrieve files
+            option :region,
+                   desc: 'The AWS region your S3 bucket resides in',
+                   type: :string, required: true
+            option :bucket,
+                   desc: 'The name of your S3 bucket',
+                   type: :string, required: true
+            option :stack,
+                   desc: 'The name of the Stack to download logs from',
+                   type: :string, required: true
+            option :decryption_keys,
+                   desc: 'The Aptible-provided keys for decryption. ' \
+                         '(Space separated if multiple)',
+                   type: :array
+
+            # For identifying files to download
+            option :string_matches,
+                   desc: 'The strings to match in log file names.' \
+                         '(Space separated if multiple)',
+                   type: :array
+            option :app_id,
+                   desc: 'The Application ID to download logs for.',
+                   type: :numeric
+            option :database_id,
+                   desc: 'The Database ID to download logs for.',
+                   type: :numeric
+            option :endpoint_id,
+                   desc: 'The Endpoint ID to download logs for.',
+                   type: :numeric
+            option :container_id,
+                   desc: 'The container ID to download logs for'
+            option :start_date,
+                   desc: 'Get logs starting from this (UTC) date ' \
+                         '(format: YYYY-MM-DD)',
+                   type: :string
+            option :end_date,
+                   desc: 'Get logs before this (UTC) date (format: YYYY-MM-DD)',
+                   type: :string
+
+            # We don't download by default
+            option :download_location,
+                   desc: 'The local path place downloaded log files. ' \
+                         'If you do not set this option, the file names ' \
+                         'will be shown, but not downloaded.',
+                   type: :string
+
+            def logs_from_archive
+              ensure_aws_creds
+              validate_log_search_options(options)
+
+              id_options = [
+                options[:app_id],
+                options[:database_id],
+                options[:endpoint_id],
+                options[:container_id]
+              ]
+
+              date_options = [options[:start_date], options[:end_date]]
+
+              r_type = 'apps' if options[:app_id]
+              r_type = 'databases' if options[:database_id]
+              r_type = 'proxy' if options[:endpoint_id]
+
+              if date_options.any?
+                start_date = utc_date(options[:start_date])
+                end_date = utc_date(options[:end_date])
+                if end_date < start_date
+                  raise Thor::Error, 'End date must be after start date.'
+                end
+                time_range = [start_date, end_date]
+                CLI.logger.info "Searching from #{start_date} to #{end_date}"
+              else
+                time_range = nil
+              end
+
+              # --string-matches is useful for matching by partial container id,
+              # or for more flexibility than the currently supported id_options
+              # may allow for. We should update id_options with new use cases,
+              # but leave string_matches as a way to download any named file
+              if options[:string_matches]
+                files = find_s3_files_by_string_match(
+                  options[:region],
+                  options[:bucket],
+                  options[:stack],
+                  options[:string_matches]
+                )
+              elsif id_options.any?
+                if options[:container_id]
+                  search_attrs = { container_id: options[:container_id] }
+                else
+                  search_attrs = { type: r_type, id: id_options.compact.first }
+                end
+                files = find_s3_files_by_attrs(
+                  options[:region],
+                  options[:bucket],
+                  options[:stack],
+                  search_attrs,
+                  time_range
+                )
+              end
+
+              unless files.any?
+                raise Thor::Error, 'No files found that matched all criteria'
+              end
+
+              CLI.logger.info "Found #{files.count} matching files..."
+
+              if options[:download_location]
+                # Since these files likely contain PHI, we will only download
+                # them if the user is explicit about where to save them.
+                files.each do |file|
+                  shasum = info_from_path(file)[:shasum]
+                  decrypt_and_translate_s3_file(
+                    file,
+                    encryption_key(shasum, options[:decryption_keys]),
+                    options[:region],
+                    options[:bucket],
+                    options[:download_location]
+                  )
+                end
+              else
+                files.each do |file|
+                  CLI.logger.info file.split('/').drop(4).join('/')
+                end
+                m = 'No files were downloaded. Please provide a location ' \
+                    'with --download-location to download the files.'
+                CLI.logger.warn m
+              end
+            end
           end
         end
       end

data/lib/aptible/cli/subcommands/maintenance.rb

@@ -0,0 +1,97 @@
+module Aptible
+  module CLI
+    module Subcommands
+      module Maintenance
+        def self.included(thor)
+          thor.class_eval do
+            include Helpers::Environment
+            include Helpers::Maintenance
+            include Helpers::Token
+
+            desc 'maintenance:apps',
+                 'List Apps impacted by maintenance schedules where '\
+                 'restarts are required'
+            option :environment
+            define_method 'maintenance:apps' do
+              found_maintenance = false
+              m = maintenance_apps
+              Formatter.render(Renderer.current) do |root|
+                root.grouped_keyed_list(
+                  { 'environment' => 'handle' },
+                  'label'
+                ) do |node|
+                  scoped_environments(options).each do |account|
+                    m.select { |app| app.account.id == account.id }
+                     .each do |app|
+                      next unless app.maintenance_deadline
+                      found_maintenance = true
+                      node.object do |n|
+                        ResourceFormatter.inject_maintenance(
+                          n,
+                          'aptible restart --app',
+                          app,
+                          account
+                        )
+                      end
+                    end
+                  end
+                end
+              end
+              if found_maintenance
+                explanation 'app'
+              else
+                no_maintenances 'app'
+              end
+            end
+            desc 'maintenance:dbs',
+                 'List Databases impacted by maintenance schedules where '\
+                 'restarts are required'
+            option :environment
+            define_method 'maintenance:dbs' do
+              found_maintenance = false
+              m = maintenance_databases
+              Formatter.render(Renderer.current) do |root|
+                root.grouped_keyed_list(
+                  { 'environment' => 'handle' },
+                  'label'
+                ) do |node|
+                  scoped_environments(options).each do |account|
+                    m.select { |db| db.account.id == account.id }
+                     .each do |db|
+                      next unless db.maintenance_deadline
+                      found_maintenance = true
+                      node.object do |n|
+                        ResourceFormatter.inject_maintenance(
+                          n,
+                          'aptible db:restart',
+                          db,
+                          account
+                        )
+                      end
+                    end
+                  end
+                end
+              end
+              if found_maintenance
+                explanation 'database'
+              else
+                no_maintenances 'database'
+              end
+            end
+          end
+        end
+
+        def explanation(resource_type)
+          CLI.logger.warn "\nYou may restart these #{resource_type}(s)"\
+                          ' at any time, or Aptible will restart it'\
+                          ' during the defined window.'
+        end
+
+        def no_maintenances(resource_type)
+          CLI.logger.info "\nNo #{resource_type}s found affected "\
+                          'by maintenance schedules.'
+        end
+      end
+    end
+  end
+end

data/lib/aptible/cli/subcommands/metric_drain.rb

@@ -5,6 +5,7 @@ module Aptible
         SITES = {
           'US1' => 'https://app.datadoghq.com',
           'US3' => 'https://us3.datadoghq.com',
+          'US5' => 'https://us5.datadoghq.com',
           'EU1' => 'https://app.datadoghq.eu',
           'US1-FED' => 'https://app.ddog-gov.com'
         }.freeze
@@ -84,6 +85,35 @@ module Aptible
               create_metric_drain(account, opts)
             end
 
+            desc 'metric_drain:create:influxdb:customv2 HANDLE '\
+                   '--org ORGANIZATION --token INFLUX_TOKEN ' \
+                   '--url URL_INCLUDING_PORT ' \
+                   '--bucket INFLUX_BUCKET_NAME ' \
+                   '--environment ENVIRONMENT',
+                 'Create an InfluxDB v2 Metric Drain'
+            option :bucket, type: :string
+            option :org, type: :string
+            option :token, type: :string
+            option :url, type: :string
+            option :environment
+            define_method 'metric_drain:create:influxdb:customv2' do |handle|
+              account = ensure_environment(options)
+
+              config = {
+                address: options[:url],
+                org: options[:org],
+                authToken: options[:token],
+                bucket: options[:bucket]
+              }
+              opts = {
+                handle: handle,
+                drain_configuration: config,
+                drain_type: :influxdb2
+              }
+
+              create_metric_drain(account, opts)
+            end
+
             desc 'metric_drain:create:datadog HANDLE '\
                  '--api_key DATADOG_API_KEY '\
                  '--site DATADOG_SITE ' \

data/lib/aptible/cli/version.rb

@@ -1,5 +1,5 @@
 module Aptible
   module CLI
-    VERSION = '0.19.4'.freeze
+    VERSION = '0.19.7'.freeze
   end
 end

data/spec/aptible/cli/helpers/date_helpers_spec.rb

@@ -0,0 +1,12 @@
+require 'spec_helper'
+
+describe Aptible::CLI::Helpers::DateHelpers do
+  subject { Class.new.send(:include, described_class).new }
+
+  describe '#utc_string' do
+    it 'should accept a Datetime string from our API and return a UTC string' do
+      result = subject.utc_string('2023-09-05T22:00:00.000Z')
+      expect(result).to eq '2023-09-05 22:00:00 UTC'
+    end
+  end
+end