bako 0.1.2 → 0.1.3

data/.ruby-version

@@ -0,0 +1 @@
+2.4.1

data/.travis.yml

@@ -1,8 +1,13 @@
 sudo: false
 language: ruby
+cache: bundler
 rvm:
 - 2.3.3
 before_install: gem install bundler -v 1.14.6
+addons:
+  code_climate:
+    repo_token:
+      secure: "rzEVWgZoxVjhiqC3YwS3h2+aYLlA8FWnEuHfrvFS7kRbiOmA0h8STR7ajMS8K4nglen1UCo5uoH194UeGNeaVKUgADS/5Db4nadA9fQ8bOwpIyq9FpAA9IHqMkmQ7cHPJ6cYf4m0z25mVq4Wy7xJQQgBSSLJPGe9ZAHrE6sHd0LggsGM4St6IYqUXyf3Nshs36Mx70Kxi0YIh743frUAWDEXWv7VPtKjhLEBHN6ZB3F9W3HvwlRrpFKx5nJT6aeEdQxy+Day3cWel6a4j/VH2pm3LDGrkF9/WByu1ULCL5aUY46vKhAfVdWIucUc9ABi09+6xhI/mf8rcMUeK96qa/gCtE1f9Tn7bX8qolvJlOprJanYTiy2eOMjP0PCuWlOazW7oKoODIn74LHIZqz9Nk3zS9GmLsfc87M1BUM4ri9txoN8RlLu6mzR4Tf6D4uceDa8VlTJLCkxRyi/CDJDwIsmyWSeBHtoRn8AfeIhrZsmCCKq3giYUJawbprZB57aQ377f5r1ZasRrdUfpMXMw/6+YVqfnLzIaFfGsS2Jz5uddSIHww9fHYPEAy/0OIPG9AgmxrKFmi4DQe5O4I1wr3yuKRpMQtqJOBNS2vvryvoNmoMLLW2ThC5D1k2ig3/+FMMYNh8hsv6nYCz1o/EDnDxJ4bQAf9ulPAc/6feyqrk="
 deploy:
   provider: rubygems
   api_key:
@@ -11,3 +16,5 @@ deploy:
   on:
     tags: true
     repo: ayemos/bako
+after_success:
+  - bundle exec codeclimate-test-reporter

data/Gemfile

@@ -2,3 +2,8 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in bako.gemspec
 gemspec
+
+group :test do
+  gem "codeclimate-test-reporter", "~> 1.0.0"
+  gem "simplecov"
+end

data/README.md

@@ -1,8 +1,10 @@
 # Bako
+[![Gem Version](https://badge.fury.io/rb/bako.svg)](https://badge.fury.io/rb/bako)
+[![Code Climate](https://codeclimate.com/github/ayemos/bako/badges/gpa.svg)](https://codeclimate.com/github/ayemos/bako)
+[![Test Coverage](https://codeclimate.com/github/ayemos/bako/badges/coverage.svg)](https://codeclimate.com/github/ayemos/bako/coverage)
+[![Issue Count](https://codeclimate.com/github/ayemos/bako/badges/issue_count.svg)](https://codeclimate.com/github/ayemos/bako)
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/bako`. To experiment with that code, run `bin/console` for an interactive prompt.
-
-TODO: Delete this and the text above, and describe your gem
+Bako is a CLI tool to manage your [AWS Batch](https://aws.amazon.com/batch/) jobs. It also has its own DSL for AWS Batch.
 
 ## Installation
 
@@ -22,7 +24,15 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+### Prepare environment
+
+Before start using Bako, you need to prepare basic environment on AWS Batch.
+We need at least
+- one Job Queue
+- one Compute Environment
+- and one [Amazon EC2 Container Registory](https://aws.amazon.com/jp/ecr/) (only if you use container image for batch implementation)
+
+## Examples
 
 ## Development
 

data/bako.gemspec

@@ -27,4 +27,5 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "aws-sdk", "~> 2"
   spec.add_dependency "thor", "~> 0.19"
+  spec.add_dependency 'activesupport', '~> 5.0'
 end

data/example/hello/hello.rb

@@ -9,6 +9,7 @@ preproc_jobs = []
 [*1..2].each do |i|
   preproc_job = job "preproc-inception-#{i}" do
     job_definition preproc_jd
+    job_queue 'bako-test-queue-001'
 
     param({
       foo: 'bar'
@@ -22,11 +23,12 @@ train_jd = job_definition 'train-inception' do
   command ['python', '/tensorflow/tensorflow/examples/learn/mnist.py']
   type 'container'
   memory 1024
-  vcpus 16
+  vcpus 4
   image 'tensorflow/tensorflow:latest-devel'
 end
 
 job 'train-inception' do
   job_definition train_jd
-  #depends_on preproc_jobs
+  job_queue 'bako-test-queue-001'
+  depends_on preproc_jobs
 end

data/lib/bako.rb

@@ -1,5 +1,8 @@
+# frozen_string_literal: true
+
 require "bako/version"
 require 'bako/common_helper'
+require 'logger'
 
 module Bako
   def Bako.logger
@@ -7,6 +10,8 @@ module Bako
   end
 end
 
+require 'bako/error'
+
 require 'bako/dsl'
 require 'bako/dsl/context'
 require 'bako/dsl/context/job'

data/lib/bako/cli.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'thor'
 
 module Bako
@@ -5,7 +7,9 @@ module Bako
     class_option :verbose, aliases: '-v', type: :boolean
     map '--version' => :print_version
 
-    desc 'run', 'run job'
+    desc 'run JOB_FILE', 'Run batch job'
+    method_option :dry_run, aliases: '-n', type: :boolean,
+      desc: 'Run job locally without call any actual APIs'
     def runjob(path)
       require 'bako/cli/run'
       Bako::CLI::Run.new(path, options, self).run

data/lib/bako/cli/run.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bako
   class CLI::Run
     include Bako::CommonHelper
@@ -9,10 +11,12 @@ module Bako
     end
 
     def run
-      dsl.job_definitions.each do |_, jd_context|
-        jd = Bako::Models::JobDefinition.from_context(jd_context)
+      dry_run = @options['dry_run']
+
+      dsl.result[:job_definitions].each do |_, jd_context|
+        jd = Bako::Models::JobDefinition.from_context(jd_context, dry_run: dry_run)
 
-        if jd.remote_exists?
+        if !dry_run && jd.remote_exists?
           y_or_n = @thor.ask("JobDefinition #{jd.name} seems to exist on remote. would you like to update it? (y/n)")
           next unless y_or_n =~ /y/i
         end
@@ -21,14 +25,14 @@ module Bako
       end
 
       jobs_to_be_run.each do |job|
-        Bako::Models::Job.from_context(job).start
+        Bako::Models::Job.from_context(job, dry_run: dry_run).start
       end
     end
 
     private
 
     def jobs_to_be_run
-      dsl.jobs.values - dsl.jobs.values.map{|j| j.depends_on_b}.flatten
+      dsl.result[:jobs].values - dsl.result[:jobs].values.map{|j| j.result[:depends_on]}.flatten
     end
 
     def dsl

data/lib/bako/common_helper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'aws-sdk'
 
 module Bako
@@ -5,12 +7,11 @@ module Bako
     DEFAULTS = {
       vcpus: 1,
       memory: 128
-    }
+    }.freeze
+
     def batch_client
       @batch_client ||= Aws::Batch::Client.new
     end
-    def remote_resource(arn)
-    end
 
     def remote_job_definition(name_or_arn, status=nil)
       remote_job_definitions(status).find{|jd|
@@ -24,12 +25,14 @@ module Bako
       }).job_definitions.map do |jd|
         Bako::Models::JobDefinition.new(
           jd.job_definition_name,
-          jd.container_properties&.command,
-          jd.container_properties&.image,
-          jd.container_properties&.job_role_arn,
-          jd.type,
-          jd.container_properties&.memory,
-          jd.container_properties&.vcpus
+          {
+            command: jd.container_properties&.command,
+            image: jd.container_properties&.image,
+            role_arn: jd.container_properties&.job_role_arn,
+            type: jd.type,
+            memory: jd.container_properties&.memory,
+            vcpus: jd.container_properties&.vcpus,
+          }
         )
       end
     end

data/lib/bako/dsl.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class Bako::DSL
   class << self
     def parse(dsl)

data/lib/bako/dsl/context.rb

@@ -1,5 +1,9 @@
+# frozen_string_literal: true
+#
+require 'active_support/core_ext/string/inflections'
+
 class Bako::DSL::Context
-  attr_reader :jobs, :job_definitions
+  attr_reader :result
 
   def self.eval(dsl)
     new do
@@ -8,31 +12,34 @@ class Bako::DSL::Context
   end
 
   def initialize(&block)
-    @jobs = {}
-    @job_definitions = {}
+
+    @result = {
+      jobs: {},
+      job_definitions: {},
+    }
 
     instance_eval(&block)
   end
 
   private
 
-  def job(name, &block)
-    name = name.to_s
+  %i[
+    job
+    job_definition
+  ].each do |context|
+    define_method(context) do |name, &block|
+      name = name.to_s
+      result_key = context.to_s.pluralize.to_sym
 
-    if @jobs[name]
-      raise "Job `#{name}` is already defined"
-    end
+      if result[result_key][name]
+        raise "#{context.to_s.capitalize} `#{name}` is already defined"
+      end
 
-    @jobs[name] = Bako::DSL::Context::Job.new(name, &block)
-  end
-
-  def job_definition(name, &block)
-    name = name.to_s
-
-    if @job_definitions[name]
-      raise "JobDefinition `#{name}` is already defined"
+      @result[result_key][name] = load_context(context).new(name, &block)
     end
+  end
 
-    @job_definitions[name] = Bako::DSL::Context::JobDefinition.new(name, &block)
+  def load_context(context)
+    Object.const_get("Bako::DSL::Context::#{context.to_s.camelize}")
   end
 end

data/lib/bako/dsl/context/job.rb

@@ -1,21 +1,41 @@
+# frozen_string_literal: true
+
 module Bako
   class DSL::Context::Job
-    attr_reader :name, :param_b, :job_definition_b, :depends_on_b, :memory_b, :vcpus_b, :job_queue_b
+    attr_reader :name, :result
 
     def initialize(name, &block)
       @name = name
+      @result = {}
 
       instance_eval(&block)
+      validate!
     end
 
     private
 
+    %i[
+      param
+      memory
+      vcpus
+      job_queue
+    ].each do |attr|
+      define_method(attr) do |v|
+        result[attr] = v
+      end
+    end
+
+    def validate!
+      raise InvalidArgumentError.new('job_queue must be set') unless result[:job_queue]
+      raise InvalidArgumentError.new('job_definition must be set') unless result[:job_definition]
+    end
+
     def job_definition(jd)
       if jd.nil?
         raise InvalidArgumentError.new('JobDefinition must be set')
       end
 
-      @job_definition_b =
+      result[:job_definition] =
         if jd.is_a?(Bako::DSL::Context::JobDefinition)
           jd
         elsif jd.is_a?(String)
@@ -26,7 +46,7 @@ module Bako
     end
 
     def depends_on(jobs)
-      @depends_on_b = jobs&.map do |job|
+      result[:depends_on] = jobs&.map do |job|
         if job.is_a?(Bako::DSL::Context::Job)
           job
         elsif job.is_a?(String)
@@ -37,20 +57,12 @@ module Bako
       end
     end
 
-    def param(h)
-      @param_b = h
-    end
-
-    def memory(memory_b)
-      @memory_b = memory_b
-    end
-
-    def vcpus(vcpus_b)
-      @vcpus_b = vcpus_b
-    end
-
-    def job_queue(job_queue_b)
-      @job_queue_b = job_queue_b
+    def command(v)
+      result[:command] = if v.is_a?(Array)
+                           v.map(&:to_s)
+                         else
+                           v.split.map(&:to_s)
+                         end
     end
   end
 end

data/lib/bako/dsl/context/job_definition.rb

@@ -1,39 +1,34 @@
+# frozen_string_literal: true
 
 module Bako
   class DSL::Context::JobDefinition
-    attr_reader :name, :type_b, :image_b, :command_b, :role_arn_b, :memory_b, :vcpus_b
+    attr_reader :name, :result
 
     def initialize(name, &block)
       @name = name
-      @type_b = 'container'
+      @result = {}
 
       instance_eval(&block)
+      validate!
     end
 
     private
 
-    def type(job_type)
-      @type_b = job_type
+    def validate!
+      raise InvalidArgumentError.new('type must be set') unless result[:type]
     end
 
-    def image(image_b)
-      @image_b = image_b
-    end
-
-    def command(command_b)
-      @command_b = command_b
-    end
-
-    def role_arn(role_arn_b)
-      @role_arn_b = role_arn_b
-    end
-
-    def memory(memory_b)
-      @memory_b = memory_b
-    end
-
-    def vcpus(vcpus_b)
-      @vcpus_b = vcpus_b
+    %i[
+      type
+      image
+      command
+      role_arn
+      memory
+      vcpus
+    ].each do |attr|
+      define_method(attr) do |v|
+        result[attr] = v
+      end
     end
   end
 end

data/lib/bako/error.rb

@@ -1,4 +1,6 @@
-module Hako
+# frozen_string_literal: true
+
+module Bako
   class InvalidArgumentError < StandardError
   end
 end

data/lib/bako/models/job.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'yaml'
 require 'pathname'
 
@@ -6,50 +8,62 @@ module Bako
     class Job
       include Bako::CommonHelper
 
-      attr_reader :name, :job_definition, :depends_on, :param, :id, :memory, :vcpus, :job_queue
+      attr_reader :name, :job_definition, :depends_on, :param, :id, :memory, :vcpus, :job_queue, :command
 
-      def self.from_context(context)
+      def self.from_context(context, dry_run: false)
         new(
           context.name,
-          context.job_definition_b,
-          context.depends_on_b&.map{|job_context|
-            Bako::Models::Job.from_context(job_context)
-          },
-          context.param_b,
-          context.memory_b,
-          context.vcpus_b,
-          context.job_queue_b
+          context.result,
+          dry_run: dry_run
         )
       end
 
-      def initialize(name, job_definition, depends_on, param, memory, vcpus, job_queue)
+      def initialize(name, result, dry_run: false)
         @name = name
-        @job_definition = job_definition
-        @depends_on = depends_on
-        @param = param
-        @memory = memory
-        @vcpus = vcpus
-        @job_queue = job_queue
+        @job_definition = result[:job_definition]
+        @depends_on = result[:depends_on]&.map{|job_context|
+          Bako::Models::Job.from_context(job_context)
+        }
+        @param = result[:param]
+        @memory = result[:memory]
+        @vcpus = result[:vcpus]
+        @job_queue = result[:job_queue]
+        @command = result[:command]
+        @dry_run = dry_run
       end
 
       def start
-        # start 'depends_on' jobs first
-        d_ids = []
+        _start(dry_run: @dry_run)
+      end
 
-        @depends_on&.each do |job|
-          d_ids << job.start.job_id
-        end
+      def _start(dry_run: false)
+        # start 'depends_on' jobs first
+        @depends_on&.each{|j| j._start(dry_run: dry_run)}
 
-        # start job
-        resp = batch_client.submit_job({
+        Bako.logger.info("Submitting job #{@name}:")
+        job_arg = {
           job_definition: @job_definition.name,
           job_name: @name,
           job_queue: @job_queue,
           depends_on: @depends_on&.map{|j| [[:job_id, j.id]].to_h},
-          parameters: @param
-        })
+          parameters: @param,
+          container_overrides: {
+            command: @command,
+            vcpus: @vcpus,
+            memory: @memory
+          }
+        }
+
+        Bako.logger.info("\n#{job_arg.to_yaml}\n")
+
+        # start job
+        if dry_run
+          @id = '(dry_run)'
+        else
+          resp = batch_client.submit_job(job_arg)
+          @id = resp.job_id
+        end
 
-        @id = resp.job_id
         Bako.logger.info("Submitted job #{@name} (id: #{@id})")
 
         resp