hako 2.16.0 → 2.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 247a60cab9d7f9bec3f63bb9858453499066c9b54d23ce9f1d8bae69c133973a
-  data.tar.gz: c004ff0c4aac3d742f89f4eeb455ff189acebabc0d398dc586eb835902129577
+  metadata.gz: bf69da97907f68236fdaeca00d74f4933d0c517afd44e072721a77b2d5ba9d0e
+  data.tar.gz: 1d3455be0daf4d3c0e61e5384fd82ccbb2a14b4a007d5115cf77b4495f63b115
 SHA512:
-  metadata.gz: c3a12ca201d09312f3c71e18e7418941aaf9330ad99d7acccb32094f0579d3085817004cea7209691d43a4f00f3181c242fbe452cc856ae9aad723bbf05d829b
-  data.tar.gz: a110da777e6ab79a3af3a3ebd6d6923e2428b59947208ad0bf771577bb45fb775ccbe723d0377d32e6b3df52419914e60ff1681a574db304763f11d13c3b6bf4
+  metadata.gz: 6c59d28f95d1be5246c4454924f8dbada59e4ffea9222eb23e684a2996161a89bcb3a33d8af1779637f3248d824118f3b48b6745816c608ae756e8646654ffc1
+  data.tar.gz: 2039675e62e6f47b3a9e37b63b843c5ac099079df435cc9f2705758722745044319f135c2b7f35371f4b192ead601d4ec842e362943cd961d2b32308909ecba1

data/.github/workflows/ci.yml

@@ -13,10 +13,10 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
-          - '2.6'
-          - '2.7'
-          - '3.0'
           - '3.1'
+          - '3.2'
+          - '3.3'
+          - '3.4'
     steps:
       - uses: actions/checkout@v2
       - uses: ruby/setup-ruby@v1
@@ -32,6 +32,6 @@ jobs:
       - uses: ruby/setup-ruby@v1
         with:
           # Use the same version with .rubocop.yml
-          ruby-version: 2.6
+          ruby-version: 3.1
           bundler-cache: true
       - run: bundle exec rubocop

data/.rubocop.yml

@@ -2,7 +2,7 @@ inherit_from: .rubocop_todo.yml
 
 AllCops:
   DisplayCopNames: true
-  TargetRubyVersion: 2.6
+  TargetRubyVersion: 3.1
   NewCops: disable
 
 Layout/FirstArgumentIndentation:
@@ -36,6 +36,8 @@ Style/PercentLiteralDelimiters:
     '%i': '[]'
 Style/RaiseArgs:
   EnforcedStyle: compact
+Style/RedundantCondition:
+  Enabled: false
 Style/SignalException:
   Enabled: false
 Style/SoleNestedConditional:

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+# 2.18.0 (2026-01-26)
+## New features
+- Support EFS
+- Support resource_requirements for GPU
+- Support SecretsManager secrets in dry-run
+- Launch dependent containers on `hako oneshot`
+- Support deployment circuit breaker
+- Support FireLens
+
+# 2.17.0 (2022-05-31)
+## New features
+- Support runtime_platform option
+
 # 2.16.0 (2022-05-02)
 ## New features
 - Support ephemeral_storage option in ECS scheduler

data/hako.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
-  spec.required_ruby_version = '>= 2.6.0'
+  spec.required_ruby_version = '>= 3.1.0'
 
   spec.add_dependency 'aws-sdk-applicationautoscaling'
   spec.add_dependency 'aws-sdk-autoscaling'
@@ -30,12 +30,14 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'aws-sdk-elasticloadbalancing'
   spec.add_dependency 'aws-sdk-elasticloadbalancingv2', '>= 1.54.0'
   spec.add_dependency 'aws-sdk-s3'
+  spec.add_dependency 'aws-sdk-secretsmanager'
   spec.add_dependency 'aws-sdk-servicediscovery'
   spec.add_dependency 'aws-sdk-sns'
   spec.add_dependency 'aws-sdk-ssm'
   spec.add_dependency 'jsonnet'
 
   spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'oga' # XML library for aws-sdk
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'rubocop', '>= 0.53.0'

data/lib/hako/container.rb

@@ -222,6 +222,29 @@ module Hako
       end
     end
 
+    # @return [Array, nil]
+    def resource_requirements
+      if @definition.key?('resource_requirements')
+        @definition['resource_requirements'].map do |rr|
+          {
+            type: rr.fetch('type'),
+            value: rr.fetch('value'),
+          }
+        end
+      end
+    end
+
+    # @return [Hash, nil]
+    def firelens_configuration
+      if @definition.key?('firelens_configuration')
+        conf = @definition['firelens_configuration']
+        {
+          type: conf.fetch('type'),
+          options: conf.fetch('options', nil),
+        }
+      end
+    end
+
     private
 
     PROVIDERS_KEY = '$providers'

data/lib/hako/definition_loader.rb

@@ -50,11 +50,20 @@ module Hako
               Container.new(@app, sidecars.fetch(name), dry_run: @dry_run)
             end
 
+          # Load linked containers
           containers[name].links.each do |link|
             m = link.match(/\A([^:]+):([^:]+)\z/)
             names << (m ? m[1] : link)
           end
 
+          # Load dependent containers
+          depends_on = containers[name].depends_on
+          if depends_on
+            depends_on.each do |depend_on|
+              names << depend_on.fetch(:container_name)
+            end
+          end
+
           containers[name].volumes_from.each do |volumes_from|
             names << volumes_from[:source_container]
           end

data/lib/hako/schedulers/ecs.rb

@@ -4,6 +4,7 @@ require 'aws-sdk-autoscaling'
 require 'aws-sdk-ec2'
 require 'aws-sdk-ecs'
 require 'aws-sdk-s3'
+require 'aws-sdk-secretsmanager'
 require 'aws-sdk-sns'
 require 'aws-sdk-ssm'
 require 'hako'
@@ -59,9 +60,21 @@ module Hako
         end
         @oneshot_notification_prefix = options.fetch('oneshot_notification_prefix', nil)
         if options.key?('deployment_configuration')
+          deployment_configuration = options.fetch('deployment_configuration')
+
           @deployment_configuration = {}
+          if deployment_configuration.key?('deployment_circuit_breaker')
+            circuit_breaker = deployment_configuration.fetch('deployment_circuit_breaker')
+            if circuit_breaker.fetch('rollback', nil)
+              validation_error!('rollback option of deployment_circuit_breaker is not supported because Hako handles rollback instead of the circuit breaker')
+            end
+            @deployment_configuration[:deployment_circuit_breaker] = {
+              enable: circuit_breaker.fetch('enable'),
+              rollback: false,
+            }
+          end
           %i[maximum_percent minimum_healthy_percent].each do |key|
-            @deployment_configuration[key] = options.fetch('deployment_configuration')[key.to_s]
+            @deployment_configuration[key] = deployment_configuration[key.to_s]
           end
         else
           @deployment_configuration = nil
@@ -81,6 +94,13 @@ module Hako
           end
         end
         @requires_compatibilities = options.fetch('requires_compatibilities', nil)
+        if options.key?('runtime_platform')
+          runtime_platform = options.fetch('runtime_platform')
+          @runtime_platform = {
+            cpu_architecture: runtime_platform.fetch('cpu_architecture', nil),
+            operating_system_family: runtime_platform.fetch('operating_system_family', nil),
+          }
+        end
         @launch_type = options.fetch('launch_type', nil)
         if options.key?('capacity_provider_strategy')
           @capacity_provider_strategy = options.fetch('capacity_provider_strategy').map do |strategy|
@@ -393,9 +413,18 @@ module Hako
         @ec2_client ||= Aws::EC2::Client.new(region: @region)
       end
 
+      # @param [String] region
       # @return [Aws::SSM::Client]
-      def ssm_client
-        @ssm_client ||= Aws::SSM::Client.new(region: @region)
+      def ssm_client(region)
+        @ssm_clients ||= {}
+        @ssm_clients[region] ||= Aws::SSM::Client.new(region: region)
+      end
+
+      # @param [String] region
+      # @return [Aws::SecretsManager::Client]
+      def secretsmanager_client(region)
+        @secretsmanager_clients ||= {}
+        @secretsmanager_clients[region] ||= Aws::SecretsManager::Client.new(region: region)
       end
 
       # @return [EcsElb, EcsElbV2]
@@ -523,6 +552,9 @@ module Hako
         if actual_definition.requires_compatibilities != @requires_compatibilities
           return true
         end
+        if actual_definition.runtime_platform != @runtime_platform
+          return true
+        end
         if actual_definition.ephemeral_storage != @ephemeral_storage
           return true
         end
@@ -566,6 +598,7 @@ module Hako
             container_definitions: definitions,
             volumes: volumes_definition,
             requires_compatibilities: @requires_compatibilities,
+            runtime_platform: @runtime_platform,
             cpu: @cpu,
             memory: @memory,
             ephemeral_storage: @ephemeral_storage,
@@ -603,6 +636,7 @@ module Hako
               container_definitions: definitions,
               volumes: volumes_definition,
               requires_compatibilities: @requires_compatibilities,
+              runtime_platform: @runtime_platform,
               cpu: @cpu,
               memory: @memory,
               ephemeral_storage: @ephemeral_storage,
@@ -640,6 +674,23 @@ module Hako
               labels: configuration['labels'],
               scope: configuration['scope'],
             }
+          elsif volume.key?('efs_volume_configuration')
+            configuration = volume['efs_volume_configuration']
+            authorization_config =
+              if configuration.key?('authorization_config')
+                ac = configuration['authorization_config']
+                {
+                  access_point_id: ac['access_point_id'],
+                  iam: ac['iam'],
+                }
+              end
+            definition[:efs_volume_configuration] = {
+              file_system_id: configuration.fetch('file_system_id'),
+              root_directory: configuration['root_directory'],
+              transit_encryption: configuration['transit_encryption'],
+              transit_encryption_port: configuration['transit_encryption_port'],
+              authorization_config: authorization_config,
+            }
           else
             # When neither 'host' nor 'docker_volume_configuration' is
             # specified, ECS API treats it as if 'host' is specified without
@@ -690,6 +741,8 @@ module Hako
           docker_security_options: container.docker_security_options,
           system_controls: container.system_controls,
           repository_credentials: container.repository_credentials,
+          resource_requirements: container.resource_requirements,
+          firelens_configuration: container.firelens_configuration,
         }
       end
 
@@ -1016,13 +1069,14 @@ module Hako
           Hako.logger.debug "  latest_event_id=#{latest_event_id}, deployments=#{s.deployments}"
           no_active = s.deployments.all? { |d| d.status != 'ACTIVE' }
           primary = s.deployments.find { |d| d.status == 'PRIMARY' }
-          if primary.desired_count * 2 < @started_task_ids.size
+          if primary.rollout_state == 'FAILED'
+            Hako.logger.error("New deployment is failing: #{primary.rollout_state_reason}")
+            report_task_diagnostics(service.cluster_arn, @started_task_ids)
+            return false
+          end
+          if primary.desired_count * 2 < @started_task_ids.size && !ecs_circuit_breaker_enabled?
             Hako.logger.error('Some started tasks are stopped. It seems new deployment is failing to start')
-            @started_task_ids.each_slice(100) do |task_ids|
-              ecs_client.describe_tasks(cluster: service.cluster_arn, tasks: task_ids).tasks.each do |task|
-                report_task_diagnostics(task)
-              end
-            end
+            report_task_diagnostics(service.cluster_arn, @started_task_ids)
             return false
           end
           primary_ready = primary && primary.running_count == primary.desired_count
@@ -1034,6 +1088,11 @@ module Hako
         end
       end
 
+      # @return [Boolean]
+      def ecs_circuit_breaker_enabled?
+        @deployment_configuration&.dig(:deployment_circuit_breaker, :enable)
+      end
+
       # @param [Array<Aws::ECS::Types::ServiceEvent>] events
       # @return [String, nil]
       def find_latest_event_id(events)
@@ -1044,20 +1103,24 @@ module Hako
         end
       end
 
-      TASK_ID_RE = /\(task ([\h-]+)\)\.\z/.freeze
+      TASK_ID_RE = /\(task ([\h-]+)\)\.\z/
       # @param [String] message
       # @return [String, nil]
       def extract_task_id(message)
         message.slice(TASK_ID_RE, 1)
       end
 
-      # @param [Aws::ECS::Types::Task] task
+      # @param [Array<String>] task_ids
       # @return [nil]
-      def report_task_diagnostics(task)
-        Hako.logger.error("task_definition_arn=#{task.task_definition_arn} last_status=#{task.last_status}")
-        Hako.logger.error("  stopped_reason: #{task.stopped_reason}")
-        task.containers.sort_by(&:name).each do |container|
-          Hako.logger.error("    Container #{container.name}: last_status=#{container.last_status} exit_code=#{container.exit_code.inspect} reason=#{container.reason.inspect}")
+      def report_task_diagnostics(cluster, task_ids)
+        task_ids.each_slice(100) do |batch|
+          ecs_client.describe_tasks(cluster: cluster, tasks: batch).tasks.each do |task|
+            Hako.logger.error("task_definition_arn=#{task.task_definition_arn} last_status=#{task.last_status}")
+            Hako.logger.error("  stopped_reason: #{task.stopped_reason}")
+            task.containers.sort_by(&:name).each do |container|
+              Hako.logger.error("    Container #{container.name}: last_status=#{container.last_status} exit_code=#{container.exit_code.inspect} reason=#{container.reason.inspect}")
+            end
+          end
         end
       end
 
@@ -1199,6 +1262,7 @@ module Hako
       # @return [nil]
       def print_volume_definition_in_cli_format(definition)
         return if definition.dig(:docker_volume_configuration, :autoprovision)
+        return if definition[:efs_volume_configuration]
         # From version 1.20.0 of ECS agent, a local volume is provisioned when
         # 'host' is specified without 'source_path'.
         return if definition.dig(:host, :source_path)
@@ -1290,6 +1354,7 @@ module Hako
               opts = dev[:host_path]
               opts += ":#{dev[:container_path]}" if dev[:container_path]
               if dev[:permissions]
+                opts += ':'
                 dev[:permissions].each do |permission|
                   opts += permission[0] if %w[read write mknod].include?(permission)
                 end
@@ -1383,22 +1448,74 @@ module Hako
         nil
       end
 
+      SecretValueFrom = Struct.new(:arn, :secret_name, :json_key, :version_stage, :version_id)
+
       # @param [Hash] container_definition
       # @return [nil]
       def check_secrets(container_definition)
-        parameter_names = (container_definition[:secrets] || []).map { |secret| secret.fetch(:value_from) }
-        invalid_parameter_names = parameter_names.each_slice(10).flat_map do |names|
-          names = names.map do |name|
-            if name.start_with?('arn:')
-              name.slice(%r{:parameter(/.+)\z}, 1)
+        parameter_arns = []
+        secret_value_arns = []
+        invalid_arns = []
+
+        (container_definition[:secrets] || []).each do |secret|
+          arn = Aws::ARNParser.parse(secret.fetch(:value_from))
+          case arn.service
+          when 'ssm'
+            parameter_arns << arn
+          when 'secretsmanager'
+            secret_value_arns << arn
+          else
+            invalid_arns << arn.to_s
+          end
+        end
+
+        parameter_arns.group_by(&:region).each do |region, region_arns|
+          region_arns.each_slice(10) do |arns|
+            invalid_arns += ssm_client(region).get_parameters(names: arns.map(&:to_s)).invalid_parameters
+          end
+        end
+
+        secret_value_arns.group_by(&:region).each do |region, region_arns|
+          secret_value_froms = region_arns.filter_map do |arn|
+            m = arn.resource.match(/\Asecret:(?<secret_name>[^:]+):(?<json_key>[^:]*):(?<version_stage>[^:]*):(?<version_id>[^:]*)\z/)
+            if m
+              f = SecretValueFrom.new(arn, m[:secret_name])
+              unless m[:json_key].empty?
+                f.json_key = m[:json_key]
+              end
+              unless m[:version_stage].empty?
+                f.version_stage = m[:version_stage]
+              end
+              unless m[:version_id].empty?
+                f.version_id = m[:version_id]
+              end
+              f
             else
-              name
+              invalid_arns << arn.to_s
+              nil
+            end
+          end
+          secret_value_froms.group_by { |f| [f.secret_name, f.version_stage, f.version_id] }.each do |(secret_name, version_stage, version_id), fs|
+            secret_arn = Aws::ARN.new(**fs[0].arn.to_h.merge(resource: "secret:#{secret_name}")).to_s
+            begin
+              secret_string = secretsmanager_client(region).get_secret_value(secret_id: secret_arn, version_stage: version_stage, version_id: version_id).secret_string
+              secret_json = nil
+              fs.each do |f|
+                if f.json_key
+                  secret_json ||= JSON.parse(secret_string)
+                  unless secret_json.key?(f.json_key)
+                    invalid_arns << f.arn.to_s
+                  end
+                end
+              end
+            rescue Aws::SecretsManager::Errors::ResourceNotFoundException
+              invalid_arns += fs.map { |f| f.arn.to_s }
             end
           end
-          ssm_client.get_parameters(names: names).invalid_parameters
         end
-        unless invalid_parameter_names.empty?
-          raise Error.new("Invalid parameters for secrets: #{invalid_parameter_names}")
+
+        unless invalid_arns.empty?
+          raise Error.new("Invalid ARNs for secrets: #{invalid_arns}")
         end
 
         nil

data/lib/hako/schedulers/ecs_definition_comparator.rb

@@ -45,8 +45,10 @@ module Hako
           struct.member(:linux_parameters, Schema::Nullable.new(linux_parameters_schema))
           struct.member(:readonly_root_filesystem, Schema::Nullable.new(Schema::Boolean.new))
           struct.member(:docker_security_options, Schema::Nullable.new(Schema::UnorderedArray.new(Schema::String.new)))
-          struct.member(:system_controls, Schema::Nullable.new(system_controls_schema))
+          struct.member(:system_controls, Schema::WithDefault.new(system_controls_schema, []))
           struct.member(:repository_credentials, Schema::Nullable.new(repository_credentials_schema))
+          struct.member(:resource_requirements, Schema::Nullable.new(Schema::UnorderedArray.new(resource_requirement_schema)))
+          struct.member(:firelens_configuration, Schema::Nullable.new(firelens_configuration_schema))
         end
       end
 
@@ -189,6 +191,20 @@ module Hako
           struct.member(:credentials_parameter, Schema::String.new)
         end
       end
+
+      def resource_requirement_schema
+        Schema::Structure.new.tap do |struct|
+          struct.member(:type, Schema::String.new)
+          struct.member(:value, Schema::String.new)
+        end
+      end
+
+      def firelens_configuration_schema
+        Schema::Structure.new.tap do |struct|
+          struct.member(:type, Schema::String.new)
+          struct.member(:options, Schema::Nullable.new(Schema::Table.new(Schema::String.new, Schema::String.new)))
+        end
+      end
     end
   end
 end

data/lib/hako/schedulers/ecs_volume_comparator.rb

@@ -22,6 +22,7 @@ module Hako
       def volume_schema
         Schema::Structure.new.tap do |struct|
           struct.member(:docker_volume_configuration, Schema::Nullable.new(docker_volume_configuration_schema))
+          struct.member(:efs_volume_configuration, Schema::Nullable.new(efs_volume_configuration_schema))
           struct.member(:host, Schema::Nullable.new(host_schema))
           struct.member(:name, Schema::String.new)
         end
@@ -37,6 +38,23 @@ module Hako
         end
       end
 
+      def efs_volume_configuration_schema
+        Schema::Structure.new.tap do |struct|
+          struct.member(:file_system_id, Schema::String.new)
+          struct.member(:root_directory, Schema::WithDefault.new(Schema::String.new, '/'))
+          struct.member(:transit_encryption, Schema::Nullable.new(Schema::String.new))
+          struct.member(:transit_encryption_port, Schema::Nullable.new(Schema::Integer.new))
+          struct.member(:authorization_config, Schema::Nullable.new(efs_authorization_config_schema))
+        end
+      end
+
+      def efs_authorization_config_schema
+        Schema::Structure.new.tap do |struct|
+          struct.member(:access_point_id, Schema::Nullable.new(Schema::String.new))
+          struct.member(:iam, Schema::Nullable.new(Schema::String.new))
+        end
+      end
+
       def host_schema
         Schema::Structure.new.tap do |struct|
           struct.member(:source_path, Schema::Nullable.new(Schema::String.new))

data/lib/hako/scripts/nginx_front.rb

@@ -10,7 +10,7 @@ module Hako
     class NginxFront < Script
       S3Config = Struct.new(:region, :bucket, :prefix) do
         # @param [Hash] options
-        def initialize(options) # rubocop:disable Lint/MissingSuper
+        def initialize(options)
           self.region = options.fetch('region')
           self.bucket = options.fetch('bucket')
           self.prefix = options.fetch('prefix', nil)

data/lib/hako/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Hako
-  VERSION = '2.16.0'
+  VERSION = '2.18.0'
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: hako
 version: !ruby/object:Gem::Version
-  version: 2.16.0
+  version: 2.18.0
 platform: ruby
 authors:
 - Kohei Suzuki
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-05-02 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-applicationautoscaling
@@ -136,6 +135,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-secretsmanager
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-servicediscovery
   requirement: !ruby/object:Gem::Requirement
@@ -206,6 +219,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: oga
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -369,7 +396,6 @@ homepage: https://github.com/eagletmt/hako
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -377,15 +403,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.6.0
+      version: 3.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
-signing_key:
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Deploy Docker container
 test_files: []