aptible-cli 0.19.3 → 0.19.6

data/lib/aptible/cli/subcommands/db.rb

@@ -84,6 +84,7 @@ module Aptible
               version = options[:version]
 
               if version && type
+                validate_image_type(type)
                 image = find_database_image(type, version)
                 db_opts[:type] = image.type
                 db_opts[:database_image] = image
@@ -91,6 +92,7 @@ module Aptible
                 raise Thor::Error, '--type is required when passing --version'
               else
                 db_opts[:type] = type || 'postgresql'
+                validate_image_type(db_opts[:type])
               end
 
               database = account.create_database!(db_opts)
@@ -346,6 +348,24 @@ module Aptible
                 end
               end
             end
+
+            desc 'db:rename OLD_HANDLE NEW_HANDLE [--environment'\
+                 ' ENVIRONMENT_HANDLE]', 'Rename a database handle. In order'\
+                 ' for the new database handle to appear in log drain and'\
+                 ' metric drain destinations, you must reload the database.'
+            option :environment
+            define_method 'db:rename' do |old_handle, new_handle|
+              env = ensure_environment(options)
+              db = ensure_database(options.merge(db: old_handle))
+              db.update!(handle: new_handle)
+              m1 = "In order for the new database name (#{new_handle}) to"\
+                   ' appear in log drain and metric drain destinations,'\
+                   ' you must reload the database.'
+              m2 = 'You can reload your database with this command: "aptible'\
+                   " db:reload #{new_handle} --environment #{env.handle}\""
+              CLI.logger.warn m1
+              CLI.logger.info m2
+            end
           end
         end
       end

data/lib/aptible/cli/subcommands/environment.rb

@@ -41,6 +41,25 @@ module Aptible
                 end
               end
             end
+
+            desc 'environment:rename OLD_HANDLE NEW_HANDLE',
+                 'Rename an environment handle. In order for the new'\
+                 ' environment handle to appear in log drain/metric'\
+                 ' destinations, you must restart the apps/databases in'\
+                 ' this environment.'
+            define_method 'environment:rename' do |old_handle, new_handle|
+              env = ensure_environment(options.merge(environment: old_handle))
+              env.update!(handle: new_handle)
+              m1 = "In order for the new environment handle (#{new_handle})"\
+                   ' to appear in log drain and metric drain destinations,'\
+                   ' you must restart the apps and databases in this'\
+                   ' environment. Also be aware of the following resources'\
+                   ' that may need names adjusted:'
+              m2 = "* Git remote URLs (ex: git@beta.aptible.com:#{new_handle}"\
+                   '/APP_HANDLE.git)'
+              m3 = '* Your own external scripts (e.g. for CI/CD)'
+              [m1, m2, m3].each { |val| CLI.logger.info val }
+            end
           end
         end
       end

data/lib/aptible/cli/subcommands/logs.rb

@@ -1,4 +1,6 @@
+require 'aws-sdk'
 require 'shellwords'
+require 'time'
 
 module Aptible
   module CLI
@@ -8,6 +10,7 @@ module Aptible
           thor.class_eval do
             include Helpers::Operation
             include Helpers::AppOrDatabase
+            include Helpers::S3LogHelpers
 
             desc 'logs [--app APP | --database DATABASE]',
                  'Follows logs from a running app or database'
@@ -25,6 +28,148 @@ module Aptible
               ENV['ACCESS_TOKEN'] = fetch_token
               exit_with_ssh_portal(op, '-o', 'SendEnv=ACCESS_TOKEN', '-T')
             end
+
+            desc 'logs_from_archive --bucket NAME --region REGION ' \
+                 '--stack NAME [ --decryption-keys ONE [OR MORE] ] ' \
+                 '[ --download-location LOCATION ] ' \
+                 '[ [ --string-matches ONE [OR MORE] ] ' \
+                 '| [ --app-id ID | --database-id ID | --endpoint-id ID | ' \
+                 '--container-id ID ] ' \
+                 '[ --start-date YYYY-MM-DD --end-date YYYY-MM-DD ] ]',
+                 'Retrieves container logs from an S3 archive in your own ' \
+                 'AWS account. You must provide your AWS credentials via ' \
+                 'the environment variables AWS_ACCESS_KEY_ID and ' \
+                 'AWS_SECRET_ACCESS_KEY'
+
+            # Required to retrieve files
+            option :region,
+                   desc: 'The AWS region your S3 bucket resides in',
+                   type: :string, required: true
+            option :bucket,
+                   desc: 'The name of your S3 bucket',
+                   type: :string, required: true
+            option :stack,
+                   desc: 'The name of the Stack to download logs from',
+                   type: :string, required: true
+            option :decryption_keys,
+                   desc: 'The Aptible-provided keys for decryption. ' \
+                         '(Space separated if multiple)',
+                   type: :array
+
+            # For identifying files to download
+            option :string_matches,
+                   desc: 'The strings to match in log file names.' \
+                         '(Space separated if multiple)',
+                   type: :array
+            option :app_id,
+                   desc: 'The Application ID to download logs for.',
+                   type: :numeric
+            option :database_id,
+                   desc: 'The Database ID to download logs for.',
+                   type: :numeric
+            option :endpoint_id,
+                   desc: 'The Endpoint ID to download logs for.',
+                   type: :numeric
+            option :container_id,
+                   desc: 'The container ID to download logs for'
+            option :start_date,
+                   desc: 'Get logs starting from this (UTC) date ' \
+                         '(format: YYYY-MM-DD)',
+                   type: :string
+            option :end_date,
+                   desc: 'Get logs before this (UTC) date (format: YYYY-MM-DD)',
+                   type: :string
+
+            # We don't download by default
+            option :download_location,
+                   desc: 'The local path place downloaded log files. ' \
+                         'If you do not set this option, the file names ' \
+                         'will be shown, but not downloaded.',
+                   type: :string
+
+            def logs_from_archive
+              ensure_aws_creds
+              validate_log_search_options(options)
+
+              id_options = [
+                options[:app_id],
+                options[:database_id],
+                options[:endpoint_id],
+                options[:container_id]
+              ]
+
+              date_options = [options[:start_date], options[:end_date]]
+
+              r_type = 'apps' if options[:app_id]
+              r_type = 'databases' if options[:database_id]
+              r_type = 'proxy' if options[:endpoint_id]
+
+              if date_options.any?
+                start_date = utc_date(options[:start_date])
+                end_date = utc_date(options[:end_date])
+                if end_date < start_date
+                  raise Thor::Error, 'End date must be after start date.'
+                end
+                time_range = [start_date, end_date]
+                CLI.logger.info "Searching from #{start_date} to #{end_date}"
+              else
+                time_range = nil
+              end
+
+              # --string-matches is useful for matching by partial container id,
+              # or for more flexibility than the currently supported id_options
+              # may allow for. We should update id_options with new use cases,
+              # but leave string_matches as a way to download any named file
+              if options[:string_matches]
+                files = find_s3_files_by_string_match(
+                  options[:region],
+                  options[:bucket],
+                  options[:stack],
+                  options[:string_matches]
+                )
+              elsif id_options.any?
+                if options[:container_id]
+                  search_attrs = { container_id: options[:container_id] }
+                else
+                  search_attrs = { type: r_type, id: id_options.compact.first }
+                end
+                files = find_s3_files_by_attrs(
+                  options[:region],
+                  options[:bucket],
+                  options[:stack],
+                  search_attrs,
+                  time_range
+                )
+              end
+
+              unless files.any?
+                raise Thor::Error, 'No files found that matched all criteria'
+              end
+
+              CLI.logger.info "Found #{files.count} matching files..."
+
+              if options[:download_location]
+                # Since these files likely contain PHI, we will only download
+                # them if the user is explicit about where to save them.
+                files.each do |file|
+                  shasum = info_from_path(file)[:shasum]
+                  decrypt_and_translate_s3_file(
+                    file,
+                    encryption_key(shasum, options[:decryption_keys]),
+                    options[:region],
+                    options[:bucket],
+                    options[:download_location]
+                  )
+                end
+              else
+                files.each do |file|
+                  CLI.logger.info file.split('/').drop(4).join('/')
+                end
+                m = 'No files were downloaded. Please provide a location ' \
+                    'with --download-location to download the files.'
+                CLI.logger.warn m
+              end
+            end
           end
         end
       end

data/lib/aptible/cli/subcommands/metric_drain.rb

@@ -8,6 +8,7 @@ module Aptible
           'EU1' => 'https://app.datadoghq.eu',
           'US1-FED' => 'https://app.ddog-gov.com'
         }.freeze
+        PATH = '/api/v1/series'.freeze
 
         def self.included(thor)
           thor.class_eval do
@@ -106,7 +107,7 @@ module Aptible
                                      "Valid options are #{sites}"
                 end
 
-                config[:series_url] = site
+                config[:series_url] = site + PATH
               end
               opts = {
                 handle: handle,

data/lib/aptible/cli/subcommands/operation.rb

@@ -16,6 +16,39 @@ module Aptible
               CLI.logger.info m
               o.update!(cancelled: true)
             end
+
+            desc 'operation:follow OPERATION_ID',
+                 'Follow logs of a running operation'
+            define_method 'operation:follow' do |operation_id|
+              o = Aptible::Api::Operation.find(operation_id, token: fetch_token)
+              raise "Operation ##{operation_id} not found" if o.nil?
+
+              if %w(failed succeeded).include? o.status
+                raise Thor::Error, "This operation has already #{o.status}. " \
+                                   'Run the following command to retrieve ' \
+                                   "the operation's logs:\n" \
+                                   "aptible operation:logs #{o.id}"
+              end
+
+              CLI.logger.info "Streaming logs for #{prettify_operation(o)}..."
+
+              attach_to_operation_logs(o)
+            end
+
+            desc 'operation:logs OPERATION_ID', 'View logs for given operation'
+            define_method 'operation:logs' do |operation_id|
+              o = Aptible::Api::Operation.find(operation_id, token: fetch_token)
+              raise "Operation ##{operation_id} not found" if o.nil?
+
+              unless %w(succeeded failed).include? o.status
+                e = 'Error - You can view the logs when operation is complete.'
+                raise Thor::Error, e
+              end
+
+              m = "Requesting operation logs for #{prettify_operation(o)}..."
+              CLI.logger.info m
+              operation_logs(o)
+            end
           end
         end
       end

data/lib/aptible/cli/version.rb

@@ -1,5 +1,5 @@
 module Aptible
   module CLI
-    VERSION = '0.19.3'.freeze
+    VERSION = '0.19.6'.freeze
   end
 end

data/spec/aptible/cli/helpers/database_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe Aptible::CLI::Helpers::Database do
+  subject { Class.new.send(:include, described_class).new }
+
+  describe '#validate_image_type' do
+    let(:pg) do
+      Fabricate(:database_image, type: 'postgresql', version: '10')
+    end
+
+    let(:redis) do
+      Fabricate(:database_image, type: 'redis', version: '9.4')
+    end
+
+    let(:token) { 'some-token' }
+
+    before do
+      allow(subject).to receive(:fetch_token).and_return(token)
+      allow(Aptible::Api::DatabaseImage).to receive(:all)
+        .and_return([pg, redis])
+    end
+
+    it 'Raises an error if provided an invalid type' do
+      bad_type = 'cassandra'
+      err = "No Database Image of type \"#{bad_type}\", " \
+            "valid types: #{pg.type}, #{redis.type}"
+      expect do
+        subject.validate_image_type(bad_type)
+      end.to raise_error(Thor::Error, err)
+    end
+
+    it 'Retruns true when provided a valid type' do
+      expect(subject.validate_image_type(pg.type)).to be(true)
+    end
+  end
+end

data/spec/aptible/cli/helpers/s3_log_helpers_spec.rb

@@ -0,0 +1,334 @@
+require 'spec_helper'
+
+describe Aptible::CLI::Helpers::S3LogHelpers do
+  subject { Class.new.send(:include, described_class).new }
+  let(:v2_pfx) { 'mystack/shareable/v2/fakesha' }
+  let(:v3_pfx) { 'mystack/shareable/v3/fakesha' }
+  let(:v2app) do
+    "#{v2_pfx}/apps-321/fakebread-json.log.2022-06-29T18:30:01.bck.gz"
+  end
+  let(:v2app_rotated) do
+    "#{v2_pfx}/apps-321/fakebread-json.1.log.2022-06-29T18:30:01.bck.gz"
+  end
+  let(:v3app) do
+    "#{v3_pfx}/apps-321/service-123/deadbeef-json.log." \
+    '2022-08-24T21:12:33.2022-08-24T21:14:38.archived.gz'
+  end
+  let(:v3db) do
+    "#{v3_pfx}/databases-321/fakebread-json.log." \
+    '2022-08-24T21:12:33.2022-08-24T21:14:38.archived.gz'
+  end
+  let(:v3db_rotated) do
+    "#{v3_pfx}/databases-321/fakebread-json.log.1." \
+    '2022-08-24T21:12:33.2022-08-24T21:14:38.archived.gz'
+  end
+
+  describe '#ensure_aws_creds' do
+    it 'Raises if no keys are provided via ENV' do
+      expect { subject.ensure_aws_creds }
+        .to raise_error(Thor::Error, /Missing environment variable/)
+    end
+
+    it 'Accepts AWS keypair from the ENV' do
+      ENV['AWS_ACCESS_KEY_ID'] = 'foo'
+      ENV['AWS_SECRET_ACCESS_KEY'] = 'bar'
+      expect { subject.ensure_aws_creds }.to_not raise_error
+    end
+  end
+
+  describe '#info_from_path' do
+    context 'time zones are in UTC' do
+      it 'processes v2 upload time in UTC' do
+        result = subject.info_from_path(v2app)
+        expect(result[:uploaded_at].zone).to eq('UTC')
+      end
+
+      it 'processes v3 log times in UTC' do
+        result = subject.info_from_path(v3app)
+        expect(result[:start_time].zone).to eq('UTC')
+        expect(result[:end_time].zone).to eq('UTC')
+      end
+    end
+
+    it 'does not choke on v3 logs with unknown timestamps' do
+      path = "#{v3_pfx}/apps-321/service-123/deadbeef-json.log." \
+             'unknown.unknown.archived.gz'
+      result = subject.info_from_path(path)
+      expect(result[:start_time]).to be(nil)
+      expect(result[:end_time]).to be(nil)
+    end
+
+    it 'can read app data from v2 paths' do
+      result = subject.info_from_path(v2app)
+      expect(result[:schema]).to eq('v2')
+      expect(result[:shasum]).to eq('fakesha')
+      expect(result[:type]).to eq('apps')
+      expect(result[:id]).to eq(321)
+      expect(result[:service_id]).to be(nil)
+      expect(result[:container_id]).to eq('fakebread')
+      expect(result[:uploaded_at]).to eq('2022-06-29T18:30:01')
+      expect(result[:container_id]).to eq('fakebread')
+      expect(result[:start_time]).to be(nil)
+      expect(result[:end_time]).to be(nil)
+    end
+
+    it 'can read app data from v3 paths' do
+      result = subject.info_from_path(v3app)
+      expect(result[:schema]).to eq('v3')
+      expect(result[:shasum]).to eq('fakesha')
+      expect(result[:type]).to eq('apps')
+      expect(result[:id]).to eq(321)
+      expect(result[:service_id]).to eq(123)
+      expect(result[:container_id]).to eq('deadbeef')
+      expect(result[:uploaded_at]).to be(nil)
+      expect(result[:start_time]).to eq('2022-08-24T21:12:33')
+      expect(result[:end_time]).to eq('2022-08-24T21:14:38')
+    end
+
+    it 'can read db data from v3 paths' do
+      result = subject.info_from_path(v3db)
+      expect(result[:schema]).to eq('v3')
+      expect(result[:shasum]).to eq('fakesha')
+      expect(result[:type]).to eq('databases')
+      expect(result[:id]).to eq(321)
+      expect(result[:service_id]).to be(nil)
+      expect(result[:container_id]).to eq('fakebread')
+      expect(result[:uploaded_at]).to be(nil)
+      expect(result[:start_time]).to eq('2022-08-24T21:12:33')
+      expect(result[:end_time]).to eq('2022-08-24T21:14:38')
+    end
+
+    context 'files  that have been rotated by docker (.json.log.1)' do
+      it 'can read data from v3 paths' do
+        result = subject.info_from_path(v3db_rotated)
+        expect(result[:schema]).to eq('v3')
+        expect(result[:shasum]).to eq('fakesha')
+        expect(result[:type]).to eq('databases')
+        expect(result[:id]).to eq(321)
+        expect(result[:service_id]).to be(nil)
+        expect(result[:container_id]).to eq('fakebread')
+        expect(result[:uploaded_at]).to be(nil)
+        expect(result[:start_time]).to eq('2022-08-24T21:12:33')
+        expect(result[:end_time]).to eq('2022-08-24T21:14:38')
+      end
+
+      it 'can read app data from v2 paths' do
+        result = subject.info_from_path(v2app)
+        expect(result[:schema]).to eq('v2')
+        expect(result[:shasum]).to eq('fakesha')
+        expect(result[:type]).to eq('apps')
+        expect(result[:id]).to eq(321)
+        expect(result[:service_id]).to be(nil)
+        expect(result[:container_id]).to eq('fakebread')
+        expect(result[:uploaded_at]).to eq('2022-06-29T18:30:01')
+        expect(result[:container_id]).to eq('fakebread')
+        expect(result[:start_time]).to be(nil)
+        expect(result[:end_time]).to be(nil)
+      end
+    end
+  end
+
+  describe '#validate_log_search_options' do
+    it 'Forces you to identify the files with a supported option' do
+      opts = {}
+      expect { subject.validate_log_search_options(opts) }
+        .to raise_error(Thor::Error, / specify an option to identify/)
+    end
+
+    it 'Does not let you pass --string-matches and id options' do
+      opts = { string_matches: ['foo'], app_id: 123 }
+      expect { subject.validate_log_search_options(opts) }
+        .to raise_error(Thor::Error, /cannot pass/)
+    end
+
+    it 'Does not let you pass multiple id options' do
+      opts = { database_id: 12, app_id: 23 }
+      expect { subject.validate_log_search_options(opts) }
+        .to raise_error(Thor::Error, /specify only one of/)
+    end
+
+    it 'Does not let you use date options with string-matches' do
+      opts = { string_matches: 12, start_date: 'foo' }
+      expect { subject.validate_log_search_options(opts) }
+        .to raise_error(Thor::Error, /cannot be used when searching by string/)
+    end
+
+    it 'Does not allow open-ended date range.' do
+      opts = { app_id: 123, start_date: 'foo' }
+      expect { subject.validate_log_search_options(opts) }
+        .to raise_error(Thor::Error, /must pass both/)
+    end
+
+    it 'Ensures you have provided a long enough container ID' do
+      opts = { container_id: 'tooshort' }
+      expect { subject.validate_log_search_options(opts) }
+        .to raise_error(Thor::Error, /at least the first 12/)
+    end
+
+    it 'Requires you to pass keys when downloading' do
+      opts = { app_id: 123, download_location: 'asdf' }
+      expect { subject.validate_log_search_options(opts) }
+        .to raise_error(Thor::Error, /You must provide decryption keys/)
+    end
+  end
+
+  describe '#find_s3_files_by_string_match' do
+    client_stub = Aws::S3::Client.new(stub_responses: true)
+    client_stub.stub_responses(
+      :list_buckets, buckets: [{ name: 'bucket' }]
+    )
+    client_stub.stub_responses(
+      :list_objects_v2, contents: [
+        { key: 'stack/it/doesnt/matter' },
+        { key: 'stack/matter/it/does/not/yoda' }
+      ]
+    )
+    before do
+      subject.stub(:s3_client) do
+        Aws::S3::Resource.new(region: 'us-east-1', client: client_stub)
+      end
+    end
+
+    it 'finds files with a single matching string' do
+      strings = %w(yoda)
+      result = subject.find_s3_files_by_string_match('us-east-1', 'bucket',
+                                                     'stack', strings)
+      expect(result).to match_array(%w(stack/matter/it/does/not/yoda))
+    end
+
+    it 'finds files with two matching strings' do
+      strings = %w(it matter)
+      result = subject.find_s3_files_by_string_match('us-east-1', 'bucket',
+                                                     'stack', strings)
+      expect(result).to match_array(%w(stack/it/doesnt/matter
+                                       stack/matter/it/does/not/yoda))
+    end
+
+    it 'only find files with all matching strings' do
+      strings = %w(it yoda)
+      result = subject.find_s3_files_by_string_match('us-east-1', 'bucket',
+                                                     'stack', strings)
+      expect(result).to match_array(%w(stack/matter/it/does/not/yoda))
+    end
+  end
+
+  describe '#find_s3_files_by_attrs' do
+    before do
+      client_stub = Aws::S3::Client.new(stub_responses: true)
+      client_stub.stub_responses(
+        :list_buckets, buckets: [{ name: 'bucket' }]
+      )
+      client_stub.stub_responses(
+        :list_objects_v2, contents: [
+          { key: v2app },
+          { key: v2app_rotated },
+          { key: v3db_rotated },
+          { key: v3db },
+          { key: v3app }
+        ]
+      )
+      subject.stub(:s3_client) do
+        Aws::S3::Resource.new(region: 'us-east-1', client: client_stub)
+      end
+    end
+
+    it 'can find apps by id' do
+      attrs = { type: 'apps', id: 321 }
+      result = subject.find_s3_files_by_attrs('us-east-1', 'bucket',
+                                              'stack', attrs)
+      expect(result).to match_array([v3app, v2app, v2app_rotated])
+    end
+
+    it 'can find databases by id' do
+      attrs = { type: 'databases', id: 321 }
+      result = subject.find_s3_files_by_attrs('us-east-1', 'bucket',
+                                              'stack', attrs)
+      expect(result).to match_array([v3db, v3db_rotated])
+    end
+
+    it 'can find by other attributes of the log file like container id' do
+      attrs = { container_id: 'deadbeef' }
+      result = subject.find_s3_files_by_attrs('us-east-1', 'bucket',
+                                              'stack', attrs)
+      expect(result).to match_array([v3app])
+    end
+  end
+
+  describe '#time_match?' do
+    # Here's a represenation of the test cases.  We keep the file timestamps
+    # fixed and move  --start-date/--end-date around to all possible combos.
+    # Note that we do foce the start to be earlier than the end, which keeps the
+    # logic here quite simple.
+
+    #      |  |se
+    #      | s|e
+    #     s|  |e
+    #      |se|
+    #     s|e |
+    #    se|  |
+
+    # s = start / lower bound of search
+    # e = end / upper bound of search
+    # |'s are the first and last timestamp in the file
+
+    let(:first_log) { Time.parse('2022-08-01T00:00:00') }
+    let(:last_log) { Time.parse('2022-09-01T00:00:00') }
+    let(:before) { Time.parse('2022-07-01T00:00:00') }
+    let(:between) { Time.parse('2022-08-15T00:00:00') }
+    let(:after) { Time.parse('2022-10-01T00:00:00') }
+
+    context 'identifies files that may have lines within a range' do
+      it 'before before does not match' do
+        range = [before, before]
+        expect(subject.time_match?(range, first_log, last_log)).to be(false)
+      end
+
+      it 'before between matches' do
+        range = [before, between]
+        expect(subject.time_match?(range, first_log, last_log)).to be(true)
+      end
+
+      it 'between between matches' do
+        range = [between, between]
+        expect(subject.time_match?(range, first_log, last_log)).to be(true)
+      end
+
+      it 'before after matches' do
+        range = [before, after]
+        expect(subject.time_match?(range, first_log, last_log)).to be(true)
+      end
+
+      it 'between after matches' do
+        range = [between, after]
+        expect(subject.time_match?(range, first_log, last_log)).to be(true)
+      end
+
+      it 'after after does not match' do
+        range = [after, after]
+        expect(subject.time_match?(range, first_log, last_log)).to be(false)
+      end
+    end
+  end
+
+  describe '#utc_date' do
+    e = 'Please provide dates in YYYY-MM-DD format'
+
+    it 'converts strings to dates in UTC' do
+      result = subject.utc_date('2022-08-30')
+      expect(result).to be_a(Time)
+      expect(result).to eq(Time.utc(2022, 8, 30, 0, 0, 0))
+    end
+
+    it 'raises an error if the input is a valid date/tiem in wrong format' do
+      expect do
+        subject.utc_date('2022-08-30 11:32')
+      end.to raise_error(Thor::Error, e)
+    end
+
+    it 'raises an error if the input is wrong' do
+      expect do
+        subject.utc_date('foobar')
+      end.to raise_error(Thor::Error, e)
+    end
+  end
+end