hako 2.6.2 → 2.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b17b6419ac68f7381855d71a4a8bb268ab1f453e148c55f655704a9d016de402
-  data.tar.gz: 4263e33011c50dac954f497a8c10744270a7e0042c25ce0550efb752114b4231
+  metadata.gz: 5e976128af20ee669b05e2e9531dc84d3b0ad46dcba305fa732cfbf0e24215dc
+  data.tar.gz: 0cde6cd9e06468444bd63a55f869424b3daae0926d6b8cd9a049e5511ba8ef30
 SHA512:
-  metadata.gz: 5c278d415a8d1bcf4c1e29346f8f298b5de39175cd5b02a7233230537881be87b7445ad4b329f1903ae9912d6f8408e7bca41d367773888323995f46977311ba
-  data.tar.gz: d44cc04c09be159f7b179f82a5b17c93fb7cd88d2c562567532b64eec72429a0625b0445769362f52802d81a7a2527d609c3ace2d1f2c325078b6a97f1e317a5
+  metadata.gz: 27279574a6d795bbe859463056f7460b501e3d2a0c38f6fa147aeafb419970ec9a02efed532fd35497f040a5fcb758145d224e6611bcc68cc003e586dc101753
+  data.tar.gz: ebb6948d88f10ff6f8ad3e7dbc3f47ea674cfb45d35b391c6611de9cbdb87bd73ea8cca02112f2dc4e27b6798a9e0308bb437aaa5db2586e2d2e747e7363ad8e

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 2.7.0 (2019-03-15)
+## New features
+- Support `entry_point` parameter
+- Support ECS Service Discovery
+  - See [examples/hello-service-discovery.jsonnet](examples/hello-service-discovery.jsonnet)
+
 # 2.6.2 (2018-12-19)
 ## Bug fixes
 - Set `platform_version` correctly

data/examples/hello-service-discovery.jsonnet

@@ -0,0 +1,49 @@
+local fileProvider = std.native('provide.file');
+local provide(name) = fileProvider(std.toString({ path: 'hello.env' }), name);
+
+{
+  scheduler: {
+    type: 'ecs',
+    region: 'ap-northeast-1',
+    cluster: 'eagletmt',
+    desired_count: 2,
+    role: 'ecsServiceRole',
+    service_discovery: [
+      {
+        container_name: 'app',
+        container_port: 80,
+        service: {
+          name: 'hello-service-discovery',
+          namespace_id: 'ns-XXXXXXXXXXXXXXXX',
+          dns_config: {
+            dns_records: [
+              {
+                type: 'SRV',
+                ttl: 60,
+              },
+            ],
+          },
+          health_check_custom_config: {
+            failure_threshold: 1,
+          },
+        },
+      },
+    ],
+  },
+  app: {
+    image: 'ryotarai/hello-sinatra',
+    memory: 128,
+    cpu: 256,
+    env: {
+      PORT: '3000',
+      MESSAGE: std.format('%s-san', provide('username')),
+    },
+    port_mappings: [
+      {
+        container_port: 3000,
+        host_port: 0,
+        protocol: 'tcp',
+      },
+    ],
+  },
+}

data/hako.gemspec

@@ -30,6 +30,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'aws-sdk-elasticloadbalancing'
   spec.add_dependency 'aws-sdk-elasticloadbalancingv2'
   spec.add_dependency 'aws-sdk-s3'
+  spec.add_dependency 'aws-sdk-servicediscovery'
   spec.add_dependency 'aws-sdk-sns'
   spec.add_dependency 'aws-sdk-ssm'
   spec.add_dependency 'jsonnet'

data/lib/hako/container.rb

@@ -27,6 +27,7 @@ module Hako
       memory_reservation
       links
       essential
+      entry_point
       command
       user
       privileged

data/lib/hako/schedulers/ecs.rb

@@ -14,6 +14,7 @@ require 'hako/schedulers/ecs_definition_comparator'
 require 'hako/schedulers/ecs_elb'
 require 'hako/schedulers/ecs_elb_v2'
 require 'hako/schedulers/ecs_service_comparator'
+require 'hako/schedulers/ecs_service_discovery'
 require 'hako/schedulers/ecs_volume_comparator'
 
 module Hako
@@ -87,6 +88,9 @@ module Hako
             }
           end
         end
+        if options['service_discovery']
+          @service_discovery = EcsServiceDiscovery.new(options.fetch('service_discovery'), @region, dry_run: @dry_run)
+        end
 
         @started_at = nil
         @container_instance_arn = nil
@@ -114,6 +118,9 @@ module Hako
             @autoscaling.apply(Aws::ECS::Types::Service.new(cluster_arn: @cluster, service_name: @app_id))
           end
           ecs_elb_client.modify_attributes
+          if @service_discovery
+            @service_discovery.apply
+          end
         else
           current_service = describe_service
           task_definition_changed, task_definition = register_task_definition(definitions)
@@ -130,12 +137,18 @@ module Hako
               @autoscaling.apply(current_service)
             end
             ecs_elb_client.modify_attributes
+            if @service_discovery
+              @service_discovery.apply
+            end
           else
             Hako.logger.info "Updated service: #{service.service_arn}"
             if @autoscaling
               @autoscaling.apply(service)
             end
             ecs_elb_client.modify_attributes
+            if @service_discovery
+              @service_discovery.apply
+            end
             unless wait_for_ready(service)
               if task_definition_changed
                 Hako.logger.error("Rolling back to #{current_service.task_definition}")
@@ -295,6 +308,13 @@ module Hako
         else
           puts 'Autoscaling: No'
         end
+
+        if service.service_registries.empty?
+          puts 'Service Discovery: No'
+        else
+          puts 'Service Discovery:'
+          @service_discovery.status(service.service_registries)
+        end
       end
 
       # @return [nil]
@@ -313,6 +333,9 @@ module Hako
             ecs_client.delete_service(cluster: service.cluster_arn, service: service.service_arn)
             Hako.logger.info "#{service.service_arn} is deleted"
           end
+          unless service.service_registries.empty?
+            @service_discovery.remove(service.service_registries)
+          end
         else
           puts "Service #{@app_id} doesn't exist"
         end
@@ -611,6 +634,7 @@ module Hako
           secrets: container.secrets,
           docker_labels: container.docker_labels,
           mount_points: container.mount_points,
+          entry_point: container.entry_point,
           command: container.command,
           privileged: container.privileged,
           linux_parameters: container.linux_parameters,
@@ -831,6 +855,7 @@ module Hako
           params[:desired_count] = current_service.desired_count
         end
         warn_placement_policy_change(current_service)
+        warn_service_registries_change(current_service)
         if service_changed?(current_service, params)
           ecs_client.update_service(params).service
         else
@@ -863,6 +888,10 @@ module Hako
           ecs_elb_client.modify_attributes
           params[:load_balancers] = [ecs_elb_client.load_balancer_params_for_service]
         end
+        if @service_discovery
+          @service_discovery.apply
+          params[:service_registries] = @service_discovery.service_registries
+        end
         ecs_client.create_service(params).service
       end
 
@@ -1203,6 +1232,9 @@ module Hako
         (definition[:docker_security_options] || []).each do |docker_security_option|
           cmd << '--security-opt' << docker_security_option
         end
+        if definition[:entry_point]
+          cmd << '--entrypoint' << definition[:entry_point]
+        end
 
         cmd << "\\\n  "
         definition.fetch(:environment).each do |env|
@@ -1283,6 +1315,16 @@ module Hako
         end
       end
 
+      # @param [Aws::ECS::Types::Service] service
+      # @return [void]
+      def warn_service_registries_change(service)
+        actual_service_registries = service.service_registries.sort_by(&:registry_arn).map(&:to_h)
+        expected_service_registries = @service_discovery&.service_registries&.sort_by { |s| s[:registry_arn] } || []
+        if actual_service_registries != expected_service_registries
+          Hako.logger.warn "Ignoring updated service_registries in the configuration, because AWS doesn't allow updating them for now."
+        end
+      end
+
       # @param [Aws::ECS::Types::TaskDefinition] task_definition
       # @param [String] target_definition
       # @return [nil]

data/lib/hako/schedulers/ecs_definition_comparator.rb

@@ -32,6 +32,7 @@ module Hako
           struct.member(:secrets, Schema::Nullable.new(Schema::UnorderedArray.new(secrets_schema)))
           struct.member(:docker_labels, Schema::Table.new(Schema::String.new, Schema::String.new))
           struct.member(:mount_points, Schema::UnorderedArray.new(mount_point_schema))
+          struct.member(:entry_point, Schema::Nullable.new(Schema::OrderedArray.new(Schema::String.new)))
           struct.member(:command, Schema::Nullable.new(Schema::OrderedArray.new(Schema::String.new)))
           struct.member(:volumes_from, Schema::UnorderedArray.new(volumes_from_schema))
           struct.member(:user, Schema::Nullable.new(Schema::String.new))

data/lib/hako/schedulers/ecs_service_discovery.rb

@@ -0,0 +1,257 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-servicediscovery'
+require 'hako'
+require 'hako/error'
+require 'hako/schedulers/ecs_service_discovery_service_comparator'
+
+module Hako
+  module Schedulers
+    class EcsServiceDiscovery
+      # @param [Array<Hash>] config
+      # @param [Boolean] dry_run
+      # @param [String] region
+      def initialize(config, region, dry_run:)
+        @region = region
+        @config = config
+        @dry_run = dry_run
+      end
+
+      # @return [void]
+      def apply
+        @config.map do |service_discovery|
+          service = service_discovery.fetch('service')
+          namespace_id = service.fetch('namespace_id')
+          namespace = get_namespace(namespace_id)
+          if !namespace
+            raise Error.new("Service discovery namespace #{namespace_id} not found")
+          elsif namespace.type != 'DNS_PRIVATE'
+            raise Error.new("ECS only supports registering a service into a private DNS namespace: #{namespace.name} (#{namespace_id})")
+          end
+
+          service_name = service.fetch('name')
+          current_service = find_service(namespace_id, service_name)
+          if !current_service
+            if @dry_run
+              Hako.logger.info("Created service discovery service #{service_name} (dry-run)")
+            else
+              current_service = create_service(service)
+              Hako.logger.info("Created service discovery service #{service_name} (#{current_service.id})")
+            end
+          else
+            if service_changed?(service, current_service)
+              if @dry_run
+                Hako.logger.info("Updated service discovery service #{service_name} (#{current_service.id}) (dry-run)")
+              else
+                update_service(current_service.id, service)
+                Hako.logger.info("Updated service discovery service #{service_name} (#{current_service.id})")
+              end
+            end
+            warn_disallowed_service_change(service, current_service)
+          end
+        end
+      end
+
+      # @return [void]
+      def status(service_registries)
+        service_registries.each do |service_registry|
+          service_id = service_registry.registry_arn.slice(%r{service/(.+)\z}, 1)
+          service = get_service(service_id)
+          next unless service
+
+          namespace = get_namespace(service.namespace_id)
+          instances = service_discovery_client.list_instances(service_id: service.id).flat_map(&:instances)
+          puts "  #{service.name}.#{namespace.name} instance_count=#{instances.size}"
+          instances.each do |instance|
+            instance_attributes = instance.attributes.map { |k, v| "#{k}=#{v}" }.join(', ')
+            puts "    #{instance.id} #{instance_attributes}"
+          end
+        end
+      end
+
+      # @return [void]
+      def remove(service_registries)
+        service_registries.each do |service_registry|
+          service_id = service_registry.registry_arn.slice(%r{service/(.+)\z}, 1)
+          service = get_service(service_id)
+          unless service
+            Hako.logger.info("Service discovery service #{service_name} (#{service_id}) doesn't exist")
+            next
+          end
+          if @dry_run
+            Hako.logger.info("Deleted service discovery service #{service.name} (#{service.id}) (dry-run)")
+          else
+            deleted = false
+            10.times do |i|
+              sleep 10 unless i.zero?
+              begin
+                service_discovery_client.delete_service(id: service.id)
+                deleted = true
+                break
+              rescue Aws::ServiceDiscovery::Errors::ResourceInUse => e
+                Hako.logger.warn("#{e.class}: #{e.message}")
+              end
+            end
+            unless deleted
+              raise Error.new("Unable to delete service discovery service #{service.name} (#{service.id})")
+            end
+
+            Hako.logger.info("Deleted service discovery service #{service.name} (#{service.id})")
+          end
+        end
+      end
+
+      # @return [Hash]
+      def service_registries
+        @config.map do |service_discovery|
+          service = service_discovery.fetch('service')
+          namespace_id = service.fetch('namespace_id')
+          service_name = service.fetch('name')
+          current_service = find_service(namespace_id, service_name)
+          unless current_service
+            raise Error.new("Service discovery service #{service_name} not found")
+          end
+
+          {
+            container_name: service_discovery['container_name'],
+            container_port: service_discovery['container_port'],
+            port: service_discovery['port'],
+            registry_arn: current_service.arn,
+          }.reject { |_, v| v.nil? }
+        end
+      end
+
+      private
+
+      # @param [String] namespace_id
+      # @param [String] service_name
+      # @return [Aws::ServiceDiscovery::Types::ServiceSummary, nil]
+      def find_service(namespace_id, service_name)
+        params = {
+          filters: [
+            name: 'NAMESPACE_ID',
+            values: [namespace_id],
+            condition: 'EQ',
+          ],
+        }
+        services = service_discovery_client.list_services(params).flat_map(&:services)
+        services.find { |service| service.name == service_name }
+      end
+
+      # @return [Aws::ServiceDiscovery::Client]
+      def service_discovery_client
+        @service_discovery_client ||= Aws::ServiceDiscovery::Client.new(region: @region)
+      end
+
+      # @param [Hash] service
+      # @return [Aws::ServiceDiscovery::Types::Service]
+      def create_service(service)
+        service_discovery_client.create_service(create_service_params(service)).service
+      end
+
+      # @param [Hash] service
+      # @return [Hash]
+      def create_service_params(service)
+        dns_config = service.fetch('dns_config')
+        params = {
+          name: service.fetch('name'),
+          namespace_id: service['namespace_id'],
+          description: service['description'],
+          dns_config: {
+            namespace_id: dns_config['namespace_id'],
+            routing_policy: dns_config.fetch('routing_policy', 'MULTIVALUE'),
+          },
+        }
+        params[:dns_config][:dns_records] = dns_config.fetch('dns_records').map do |dns_record|
+          {
+            type: dns_record.fetch('type'),
+            ttl: dns_record.fetch('ttl'),
+          }
+        end
+        if (health_check_custom_config = service['health_check_custom_config'])
+          params[:health_check_custom_config] = {
+            failure_threshold: health_check_custom_config['failure_threshold'],
+          }
+        end
+        params
+      end
+
+      # @param [Hash] expected_service
+      # @param [Aws::ServiceDiscovery::Types::ServiceSummary] actual_service
+      # @return [Boolean]
+      def service_changed?(expected_service, actual_service)
+        EcsServiceDiscoveryServiceComparator.new(update_service_params(expected_service)).different?(actual_service)
+      end
+
+      # @param [String] service_id
+      # @param [Hash] service
+      def update_service(service_id, service)
+        operation_id = service_discovery_client.update_service(
+          id: service_id,
+          service: update_service_params(service),
+        ).operation_id
+        operation = wait_for_operation(operation_id)
+        if operation.status != 'SUCCESS'
+          raise Error.new("Unable to update service discovery service (#{operation.error_code}): #{operation.error_message}")
+        end
+      end
+
+      # @param [Hash] service
+      # @return [Hash]
+      def update_service_params(service)
+        dns_config = service.fetch('dns_config')
+        params = {
+          description: service['description'],
+          dns_config: {},
+        }
+        params[:dns_config][:dns_records] = dns_config.fetch('dns_records').map do |dns_record|
+          {
+            type: dns_record.fetch('type'),
+            ttl: dns_record.fetch('ttl'),
+          }
+        end
+        params
+      end
+
+      # @param [String] service_id
+      # @return [Aws::ServiceDiscovery::Types::GetOperationResponse]
+      def wait_for_operation(operation_id)
+        loop do
+          operation = service_discovery_client.get_operation(operation_id: operation_id).operation
+          return operation if %w[SUCCESS FAIL].include?(operation.status)
+
+          sleep 10
+        end
+      end
+
+      # @param [String] service_id
+      # @return [Aws::ServiceDiscovery::Types::Service, nil]
+      def get_service(service_id)
+        service_discovery_client.get_service(id: service_id).service
+      rescue Aws::ServiceDiscovery::Errors::ServiceNotFound
+        nil
+      end
+
+      # @param [String] namespace_id
+      # @return [Aws::ServiceDiscovery::Types::Namespace, nil]
+      def get_namespace(namespace_id)
+        service_discovery_client.get_namespace(id: namespace_id).namespace
+      rescue Aws::ServiceDiscovery::Errors::NamespaceNotFound
+        nil
+      end
+
+      # @param [Hash] expected_service
+      # @param [Aws::ServiceDiscovery::Types::ServiceSummary] actual_service
+      # @return [void]
+      def warn_disallowed_service_change(expected_service, actual_service)
+        expected_service = create_service_params(expected_service)
+        if expected_service.dig(:dns_config, :routing_policy) != actual_service.dns_config.routing_policy
+          Hako.logger.warn("Ignoring updated service_discovery.dns_config.routing_policy in the configuration, because AWS doesn't allow updating it for now.")
+        end
+        if expected_service[:health_check_custom_config] != actual_service.health_check_custom_config&.to_h
+          Hako.logger.warn("Ignoring updated service_discovery.health_check_custom_config in the configuration, because AWS doesn't allow updating it for now.")
+        end
+      end
+    end
+  end
+end

data/lib/hako/schedulers/ecs_service_discovery_service_comparator.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'hako/schema'
+
+module Hako
+  module Schedulers
+    class EcsServiceDiscoveryServiceComparator
+      # @param [Hash] expected_service
+      def initialize(expected_service)
+        @expected_service = expected_service
+        @schema = service_schema
+      end
+
+      # @param [Aws::ServiceDiscovery::Types::ServiceSummary] actual_service
+      # @return [Boolean]
+      def different?(actual_service)
+        !@schema.same?(actual_service.to_h, @expected_service)
+      end
+
+      private
+
+      def service_schema
+        Schema::Structure.new.tap do |struct|
+          struct.member(:description, Schema::Nullable.new(Schema::String.new))
+          struct.member(:dns_config, dns_config_schema)
+        end
+      end
+
+      def dns_config_schema
+        Schema::Structure.new.tap do |struct|
+          struct.member(:dns_records, Schema::UnorderedArray.new(dns_records_schema))
+        end
+      end
+
+      def dns_records_schema
+        Schema::Structure.new.tap do |struct|
+          struct.member(:ttl, Schema::Integer.new)
+          struct.member(:type, Schema::String.new)
+        end
+      end
+    end
+  end
+end

data/lib/hako/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Hako
-  VERSION = '2.6.2'
+  VERSION = '2.7.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hako
 version: !ruby/object:Gem::Version
-  version: 2.6.2
+  version: 2.7.0
 platform: ruby
 authors:
 - Kohei Suzuki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-12-19 00:00:00.000000000 Z
+date: 2019-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-applicationautoscaling
@@ -136,6 +136,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-servicediscovery
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sns
   requirement: !ruby/object:Gem::Requirement
@@ -312,6 +326,7 @@ files:
 - examples/hello-lb.jsonnet
 - examples/hello-nofront.jsonnet
 - examples/hello-privileged-app.jsonnet
+- examples/hello-service-discovery.jsonnet
 - examples/hello.env
 - examples/hello.jsonnet
 - examples/put-ecs-container-status-to-s3/index.js
@@ -341,6 +356,8 @@ files:
 - lib/hako/schedulers/ecs_elb.rb
 - lib/hako/schedulers/ecs_elb_v2.rb
 - lib/hako/schedulers/ecs_service_comparator.rb
+- lib/hako/schedulers/ecs_service_discovery.rb
+- lib/hako/schedulers/ecs_service_discovery_service_comparator.rb
 - lib/hako/schedulers/ecs_volume_comparator.rb
 - lib/hako/schema.rb
 - lib/hako/schema/boolean.rb
@@ -380,7 +397,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.7.6.2
 signing_key: 
 specification_version: 4
 summary: Deploy Docker container