hako 1.9.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be60277c499933be1ac03eed32027844eb410bb92d52355984833f16d19ac882
-  data.tar.gz: f07f5c690efcc8c81a1610ff9077dcf36f21cb41eb6f6ff07b475083bfbb999d
+  metadata.gz: 85ef57c377dafbf524ddb4d442108f92aee5f278d2506c5f91ca61df1541dd2b
+  data.tar.gz: 7de25a114cf37299e18ffa034d8eed56d6253857c67865bd0a15f90c1c87bb02
 SHA512:
-  metadata.gz: 17b332ddc9a4f4eca98938d7e5fec30898d6b50971f6700bb5e1717a0c211b88d97ed4b5dc9c24a4d75bb4fabc27d5d1ed374b3d51410305b8e184b5250b1a1c
-  data.tar.gz: 288283a14dbed0e30066da6b1a7d4fa6568ceda09a4272981304e6b421f91c2fce8d6b67fc535e8440cfccf51852babc71b803a8a21542ed48d2a84b59cb58e1
+  metadata.gz: a30d023690fc56f462e230b0e9b229c63091489446ac85c45a01d7158d0ba363f999ea28ca0cac2a2c9c22dfb387f3a7d5cc8b5fcd9a552fb98581f518eedeee
+  data.tar.gz: 2095d02a1a7da744416a599abf5ba16762206349c4dfd772d7d74f7ef030aab4ffa4a6a4d36005f68d42fdced3e82c2bdbef51b4a0ea1839cdd4b07432709da2

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 2.0.0 (2017-12-13)
+## New features
+- Support Jsonnet as the definition file format
+    - See [docs/jsonnet.md](docs/jsonnet.md)
+    - YAML definitions continues to be supported by hako at least v2.x series. Currently I don't have a plan which will be the default format.
+
 # 1.9.0 (2017-12-06)
 ## New features
 - Add support for awsvpc network mode

data/README.md

@@ -23,16 +23,16 @@ Or install it yourself as:
 ## Usage
 
 ```
-% hako deploy examples/hello.yml
+% hako deploy examples/hello.jsonnet
 I, [2015-10-02T12:51:24.530274 #7988]  INFO -- : Registered task-definition: arn:aws:ecs:ap-northeast-1:XXXXXXXXXXXX:task-definition/hello:29
 I, [2015-10-02T12:51:24.750501 #7988]  INFO -- : Uploaded front configuration to s3://nanika/hako/front_config/hello.conf
 I, [2015-10-02T12:51:24.877409 #7988]  INFO -- : Updated service: arn:aws:ecs:ap-northeast-1:XXXXXXXXXXXX:service/hello
 I, [2015-10-02T12:56:07.284874 #7988]  INFO -- : Deployment completed
 
-% hako deploy examples/hello.yml
+% hako deploy examples/hello.jsonnet
 I, [2015-10-02T12:56:12.262760 #8141]  INFO -- : Deployment isn't needed
 
-% hako status examples/hello.yml
+% hako status examples/hello.jsonnet
 Load balancer:
   hako-hello-XXXXXXXXXX.ap-northeast-1.elb.amazonaws.com:80 -> front:80
 Deployments:
@@ -44,7 +44,7 @@ Events:
   2015-10-05 13:35:53 +0900: (service hello) has reached a steady state.
   2015-10-05 13:35:14 +0900: (service hello) stopped 1 running tasks.
 
-% hako rollback examples/hello.yml
+% hako rollback examples/hello.jsonnet
 I, [2016-05-02T13:07:12.679926 #10961]  INFO -- : Current task defintion is hello:29. Rolling back to arn:aws:ecs:ap-northeast-1:XXXXXXXXXXXX:task-definition/hello:28
 I, [2016-05-02T13:07:12.959116 #10961]  INFO -- : Updated service: arn:aws:ecs:ap-northeast-1:XXXXXXXXXXXX:service/hello
 I, [2016-05-02T13:08:27.280686 #10961]  INFO -- : Deployment completed
@@ -53,7 +53,7 @@ I, [2016-05-02T13:08:27.280686 #10961]  INFO -- : Deployment completed
 Run oneshot command.
 
 ```
-% hako oneshot examples/hello.yml date
+% hako oneshot examples/hello.jsonnet date
 I, [2017-07-18T18:14:06.099763 #6627]  INFO -- : Task definition isn't changed: arn:aws:ecs:ap-northeast-1:XXXXXXXXXXXX:task-definition/hello-oneshot:32
 I, [2017-07-18T18:14:06.147062 #6627]  INFO -- : Started task: arn:aws:ecs:ap-northeast-1:XXXXXXXXXXXX:task/01234567-89ab-cdef-0123-456789abcdef
 I, [2017-07-18T18:14:06.193860 #6627]  INFO -- : Container instance is arn:aws:ecs:ap-northeast-1:XXXXXXXXXXXX:container-instance/01234567-89ab-cdef-0123-456789abcdef (i-0123456789abcdef0)

data/docs/jsonnet.md

@@ -0,0 +1,48 @@
+Jsonnet is a domain specific configuration language that helps you define JSON data.
+http://jsonnet.org/
+
+In hako, you can use Jsonnet format for defining applications.
+
+# External variables
+Jsonnet provide a way to refer external variables by `std.extVar(x)`.
+http://jsonnet.org/docs/stdlib.html#ext_vars
+
+## appId
+`std.extVar('appId')` returns application id of the definition.
+
+### Example
+```
+% cat awesome-app.jsonnet
+{
+  id: std.extVar('appId'),
+}
+% hako show-definition awesome-app.jsonnet
+---
+id: awesome-app
+```
+
+# Native functions
+Jsonnet provide a way to refer native functions by `std.native(x)`.
+
+## provide.$NAME
+`std.native('provide.$NAME')` returns a function which returns a corresponding EnvProvider.
+
+### Example
+```
+% cat awesome-app.env
+username=eagletmt
+% cat awesome-app.yml
+password: hako
+% cat awesome-app.jsonnet
+local provideFromFile(name) = std.native('provide.file')(std.toString({ path: 'awesome-app.env' }), name);
+local provideFromYaml(name) = std.native('provide.yaml')(std.toString({ path: 'awesome-app.yml' }), name);
+
+{
+  username: provideFromFile('username'),
+  password: provideFromYaml('password'),
+}
+% hako show-definition --expand awesome-app.jsonnet
+---
+password: hako
+username: eagletmt
+```

data/examples/front.libsonnet

@@ -0,0 +1,8 @@
+{
+  type: 'nginx_front',
+  s3: {
+    region: 'ap-northeast-1',
+    bucket: 'nanika',
+    prefix: 'hako/front_config',
+  },
+}

data/examples/hello-fargate.jsonnet

@@ -0,0 +1,84 @@
+local appId = std.extVar('appId');
+
+local region = 'us-east-1';
+local publicSubnets = ['subnet-xxxxxxxx', 'subnet-yyyyyyyy'];
+local privateSubnets = ['subnet-zzzzzzzz', 'subnet-wwwwwwww'];
+local elbSecurityGroups = ['sg-xxxxxxxx'];
+local taskSecurityGroups = ['sg-yyyyyyyy', 'sg-zzzzzzzz'];
+local awslogs = {
+    log_driver: 'awslogs',
+    options: {
+        'awslogs-group': std.format('/ecs/%s', appId),
+        'awslogs-region': region,
+        'awslogs-stream-prefix': 'ecs',
+    },
+};
+
+{
+    scheduler: {
+        type: 'ecs',
+        region: region,
+        cluster: 'eagletmt',
+        desired_count: 1,
+        task_role_arn: 'arn:aws:iam::012345678901:role/EcsDefault',
+        elb_v2: {
+            vpc_id: 'vpc-xxxxxxxx',
+            health_check_path: '/site/sha',
+            listeners: [
+                {
+                    port: 80,
+                    protocol: 'HTTP',
+                },
+                {
+                    port: 443,
+                    protocol: 'HTTPS',
+                    certificate_arn: 'arn:aws:acm:us-east-1:012345678901:certificate/01234567-89ab-cdef-0123-456789abcdef',
+                },
+            ],
+            subnets: publicSubnets,
+            security_groups: elbSecurityGroups,
+            load_balancer_attributes: {
+                'access_logs.s3.enabled': 'true',
+                'access_logs.s3.bucket': 'hako-access-logs',
+                'access_logs.s3.prefix': std.format('hako-%s', appId),
+                'idle_timeout.timeout_seconds': '5',
+            },
+            target_group_attributes: {
+                'deregistration_delay.timeout_seconds': '20',
+            },
+        },
+        # Fargate
+        execution_role_arn: 'arn:aws:iam::012345678901:role/ecsTaskExecutionRole',
+        cpu: '1024',
+        memory: '2048',
+        requires_compatibilities: ['FARGATE'],
+        network_mode: 'awsvpc',
+        launch_type: 'FARGATE',
+        network_configuration: {
+            awsvpc_configuration: {
+                subnets: privateSubnets,
+                security_groups: taskSecurityGroups,
+                assign_public_ip: 'DISABLED',
+            },
+        },
+    },
+    app: {
+        image: 'ryotarai/hello-sinatra',
+        cpu: 128,
+        memory: 256,
+        env: {
+            PORT: '3000',
+            MESSAGE: 'Hello, Fargate',
+        },
+        log_configuration: awslogs,
+    },
+    additional_containers: {
+        front: {
+            image_tag: 'hako-nginx',
+            log_configuration: awslogs,
+        },
+    },
+    scripts: [
+        (import 'front.libsonnet') + { backend_port: 3000 },
+    ],
+}

data/examples/hello.jsonnet

@@ -0,0 +1,45 @@
+local fileProvider = std.native('provide.file');
+local provide(name) = fileProvider(std.toString({ path: 'hello.env' }), name);
+
+{
+    scheduler: {
+        type: 'ecs',
+        region: 'ap-northeast-1',
+        cluster: 'eagletmt',
+        desired_count: 2,
+        task_role_arn: 'arn:aws:iam::012345678901:role/HelloRole',
+        deployment_configuration: {
+            maximum_percent: 200,
+            minimum_healthy_percent: 50,
+        },
+    },
+    app: {
+        image: 'ryotarai/hello-sinatra',
+        memory: 128,
+        cpu: 256,
+        links: [
+            'redis:redis',
+        ],
+        env: {
+            PORT: '3000',
+            MESSAGE: std.format('%s-san', provide('username')),
+        },
+    },
+    additional_containers: {
+        front: {
+            image_tag: 'hako-nginx',
+            memory: 32,
+            cpu: 32,
+        },
+        redis: {
+            image_tag: 'redis:3.0',
+            cpu: 64,
+            memory: 512,
+        },
+    },
+    scripts: [
+        (import 'front.libsonnet') + {
+            backend_port: 3000,
+        },
+    ],
+}

data/hako.gemspec

@@ -31,6 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'aws-sdk-elasticloadbalancingv2'
   spec.add_dependency 'aws-sdk-s3'
   spec.add_dependency 'aws-sdk-sns'
+  spec.add_dependency 'jsonnet'
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'coveralls'

data/lib/hako/application.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'hako/error'
+require 'hako/jsonnet_loader'
 require 'hako/yaml_loader'
 require 'pathname'
 
@@ -9,15 +11,23 @@ module Hako
     #   @return [String]
     # @!attribute [r] root_path
     #   @return [Pathname]
-    # @!attribute [r] yaml
+    # @!attribute [r] definition
     #   @return [Hash]
-    attr_reader :id, :root_path, :yaml
+    attr_reader :id, :root_path, :definition
 
-    def initialize(yaml_path)
+    def initialize(yaml_path, expand_variables: true)
       path = Pathname.new(yaml_path)
       @id = path.basename.sub_ext('').to_s
       @root_path = path.parent
-      @yaml = YamlLoader.new.load(path)
+      @definition =
+        case path.extname
+        when '.yml', '.yaml'
+          YamlLoader.new.load(path)
+        when '.jsonnet', '.json'
+          JsonnetLoader.new(self, expand_variables).load(path)
+        else
+          raise Error.new("Unknown extension: #{path}")
+        end
     end
   end
 end

data/lib/hako/cli.rb

@@ -10,7 +10,7 @@ module Hako
       deploy
       rollback
       oneshot
-      show-yaml
+      show-definition
       status
       remove
       stop
@@ -181,17 +181,19 @@ module Hako
       end
     end
 
-    class ShowYaml
+    class ShowDefinition
       def run(argv)
         parse!(argv)
-        require 'hako/yaml_loader'
-        puts YamlLoader.new.load(Pathname.new(@yaml_path)).to_yaml
+        require 'hako/application'
+        app = Application.new(@path, expand_variables: @expand_variables)
+        puts app.definition.to_yaml
       end
 
       def parse!(argv)
+        @expand_variables = false
         parser.parse!(argv)
-        @yaml_path = argv.first
-        if @yaml_path.nil?
+        @path = argv.first
+        if @path.nil?
           puts parser.help
           exit 1
         end
@@ -199,8 +201,9 @@ module Hako
 
       def parser
         @parser ||= OptionParser.new do |opts|
-          opts.banner = 'hako show-yaml FILE'
+          opts.banner = 'hako show-definition FILE'
           opts.version = VERSION
+          opts.on('--expand', 'Expand variables (Jsonnet only)') { @expand_variables = true }
         end
       end
     end

data/lib/hako/commander.rb

@@ -20,9 +20,9 @@ module Hako
     # @return [nil]
     def deploy(force: false, tag: 'latest', dry_run: false, timeout:)
       containers = load_containers(tag, dry_run: dry_run)
-      scripts = @app.yaml.fetch('scripts', []).map { |config| load_script(config, dry_run: dry_run) }
-      volumes = @app.yaml.fetch('volumes', {})
-      scheduler = load_scheduler(@app.yaml['scheduler'], scripts, volumes: volumes, force: force, dry_run: dry_run, timeout: timeout)
+      scripts = @app.definition.fetch('scripts', []).map { |config| load_script(config, dry_run: dry_run) }
+      volumes = @app.definition.fetch('volumes', {})
+      scheduler = load_scheduler(@app.definition['scheduler'], scripts, volumes: volumes, force: force, dry_run: dry_run, timeout: timeout)
 
       scripts.each { |script| script.deploy_starting(containers) }
       scheduler.deploy(containers)
@@ -33,8 +33,8 @@ module Hako
     # @param [Boolean] dry_run
     # @return [nil]
     def rollback(dry_run: false)
-      scripts = @app.yaml.fetch('scripts', []).map { |config| load_script(config, dry_run: dry_run) }
-      scheduler = load_scheduler(@app.yaml['scheduler'], scripts, dry_run: dry_run)
+      scripts = @app.definition.fetch('scripts', []).map { |config| load_script(config, dry_run: dry_run) }
+      scheduler = load_scheduler(@app.definition['scheduler'], scripts, dry_run: dry_run)
 
       scripts.each(&:rollback_starting)
       scheduler.rollback
@@ -49,9 +49,9 @@ module Hako
     # @return [nil]
     def oneshot(commands, tag:, containers:, env: {}, dry_run: false, no_wait: false)
       containers = load_containers(tag, dry_run: dry_run, with: containers)
-      scripts = @app.yaml.fetch('scripts', []).map { |config| load_script(config, dry_run: dry_run) }
-      volumes = @app.yaml.fetch('volumes', {})
-      scheduler = load_scheduler(@app.yaml['scheduler'], scripts, volumes: volumes, dry_run: dry_run)
+      scripts = @app.definition.fetch('scripts', []).map { |config| load_script(config, dry_run: dry_run) }
+      volumes = @app.definition.fetch('volumes', {})
+      scheduler = load_scheduler(@app.definition['scheduler'], scripts, volumes: volumes, dry_run: dry_run)
 
       scripts.each { |script| script.oneshot_starting(containers) }
       exit_code = with_oneshot_signal_handlers(scheduler) do
@@ -63,19 +63,19 @@ module Hako
 
     # @return [nil]
     def status
-      load_scheduler(@app.yaml['scheduler'], [], dry_run: false).status
+      load_scheduler(@app.definition['scheduler'], [], dry_run: false).status
     end
 
     # @param [Boolean] dry_run
     # @return [nil]
     def remove(dry_run:)
-      scripts = @app.yaml.fetch('scripts', []).map { |config| load_script(config, dry_run: dry_run) }
-      load_scheduler(@app.yaml['scheduler'], scripts, dry_run: dry_run).remove
+      scripts = @app.definition.fetch('scripts', []).map { |config| load_script(config, dry_run: dry_run) }
+      load_scheduler(@app.definition['scheduler'], scripts, dry_run: dry_run).remove
       scripts.each(&:after_remove)
     end
 
     def stop(dry_run:)
-      load_scheduler(@app.yaml['scheduler'], [], dry_run: dry_run).stop
+      load_scheduler(@app.definition['scheduler'], [], dry_run: dry_run).stop
     end
 
     private

data/lib/hako/definition_loader.rb

@@ -18,7 +18,7 @@ module Hako
     # @param [Array<String>, nil] with
     # @return [Hash<String, Container>]
     def load(tag, with: nil)
-      additional_containers = @app.yaml.fetch('additional_containers', {})
+      additional_containers = @app.definition.fetch('additional_containers', {})
       container_names = ['app']
       if with
         container_names.concat(with)
@@ -43,7 +43,7 @@ module Hako
           containers[name] =
             case name
             when 'app'
-              AppContainer.new(@app, @app.yaml['app'].merge('tag' => tag), dry_run: @dry_run)
+              AppContainer.new(@app, @app.definition['app'].merge('tag' => tag), dry_run: @dry_run)
             else
               Container.new(@app, additional_containers.fetch(name), dry_run: @dry_run)
             end

data/lib/hako/jsonnet_loader.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'hako'
+require 'hako/env_providers'
+require 'hako/loader'
+require 'json'
+require 'jsonnet'
+
+module Hako
+  class JsonnetLoader
+    # @param [Application] application
+    # @param [Boolean] expand_variables
+    def initialize(application, expand_variables)
+      @vm = Jsonnet::VM.new
+      @root_path = application.root_path
+      define_provider_functions(expand_variables)
+      @vm.ext_var('appId', application.id)
+    end
+
+    # @param [Pathname] path
+    def load(path)
+      JSON.parse(@vm.evaluate_file(path.to_s))
+    end
+
+    private
+
+    def define_provider_functions(expand_variables)
+      Gem.loaded_specs.each do |gem_name, spec|
+        spec.require_paths.each do |path|
+          Dir.glob(File.join(spec.full_gem_path, path, 'hako/env_providers/*.rb')).each do |provider_path|
+            provider_name = File.basename(provider_path, '.rb')
+            provider_class = Loader.new(Hako::EnvProviders, 'hako/env_providers').load(provider_name)
+            Hako.logger.debug("Loaded #{provider_class} from '#{gem_name}' gem")
+            @vm.define_function("provide.#{provider_name}") do |options, name|
+              if expand_variables
+                provider_class.new(@root_path, JSON.parse(options)).ask([name]).fetch(name)
+              else
+                "\#{#{name}}"
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hako/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Hako
-  VERSION = '1.9.0'
+  VERSION = '2.0.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hako
 version: !ruby/object:Gem::Version
-  version: 1.9.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Kohei Suzuki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-12-06 00:00:00.000000000 Z
+date: 2017-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-applicationautoscaling
@@ -150,6 +150,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: jsonnet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -270,19 +284,23 @@ files:
 - bin/console
 - bin/setup
 - docs/ecs-task-notification.md
+- docs/jsonnet.md
 - examples/create_aws_cloud_watch_logs_log_group.yml
+- examples/front.libsonnet
 - examples/front.yml
 - examples/hello-autoscaling-group.yml
 - examples/hello-autoscaling.yml
 - examples/hello-awslogs-driver.yml
 - examples/hello-cap-add-app.yml
 - examples/hello-fargate-batch.yml
+- examples/hello-fargate.jsonnet
 - examples/hello-fargate.yml
 - examples/hello-lb-v2.yml
 - examples/hello-lb.yml
 - examples/hello-nofront.yml
 - examples/hello-privileged-app.yml
 - examples/hello.env
+- examples/hello.jsonnet
 - examples/hello.yml
 - examples/put-ecs-container-status-to-s3/index.js
 - examples/put-ecs-container-status-to-s3/package.json
@@ -301,6 +319,7 @@ files:
 - lib/hako/env_providers/file.rb
 - lib/hako/env_providers/yaml.rb
 - lib/hako/error.rb
+- lib/hako/jsonnet_loader.rb
 - lib/hako/loader.rb
 - lib/hako/scheduler.rb
 - lib/hako/schedulers.rb