bako 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e72cd255a4ab160b87b7f384d827c3e57df305e6
+  data.tar.gz: ccd782ee6f831fffa2dac9283ad4f76b066da12a
+SHA512:
+  metadata.gz: eee8684dbc45c6de12a22efe4dacad56ee875c07c597598276cdefbcac6c805eed508b080b87c3e09efc2513155d764e9c13cff5387dcf06648f7e98e4b7f7c8
+  data.tar.gz: 15e61930ed78e3b3d444c69857729a77a5feb5901561529a696d0842094611772afe1ee7d5aacd72f10725ca6fef872b9213b2a06db542bad705e6dca3068d7c

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,13 @@
+sudo: false
+language: ruby
+rvm:
+- 2.3.3
+before_install: gem install bundler -v 1.14.6
+deploy:
+  provider: rubygems
+  api_key:
+    secure: bVmhqQi0lR5UjoAPJsnhZPFcBYggOmQIlVZLV515US0qEwP7lDdOLY9arWLOubfd0Kr7UXp15a2JS5IiSMw9TcAKIiAeTAm9sbtha2R0qaPGJTnmE4pkn07aef3u/zg6zXdT8kj52142sket+fp//mJgKGPrlN41Tx+3xBC5+tZWQFOciUTKSz4+rpY9xgI6RECp+P5TLoopowCPvZpC3z4N5XVzzlvL+JMVnBEEc6+qXRNieasrwxzhOkY7qTj3x/tSDMjMnv+CMDUZrxVrwARZv1MJRL1alN/pCTsQr6e6iL7iBfvLL2Iy02GX1gRCi61MCJltjdPhQWm7mjV8+G4AwGVkrkJwMD5yBgt9g79lahLv9v0KIf8sxK7cMg0jYKW+apl8/hctZnkI9dy8eSw500F2obCg/iEznz3+t16NiMoBtD8Dk7Etg6DSnuiPzCGlRAq5zDIx5YRzl+1NyAmnGSgcmQkF82dKEkOyKUVLqWGJ9J+O0sFMnAOk0Lw5TaYjd2+1CzuXPmXnN+wgRxnCQKtZdVZ77PhL+NXihcvRHew8nkW0DRLe7k2qI1AQzQrSk0v13LE4mlm8ZhcOTDxbFqGn9+p7HmzWZX3dbxBb3dsuBUbhf1JBBvcbiar1CUf59LUzlJ8xADOOqbbb4uo4Z/TGb1y5UAXYSRuMpnQ=
+  gem: bako
+  on:
+    tags: true
+    repo: ayemos/bako

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in bako.gemspec
+gemspec

data/README.md

@@ -0,0 +1,36 @@
+# Bako
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/bako`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'bako'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install bako
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/bako.
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bako.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'bako/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "bako"
+  spec.version       = Bako::VERSION
+  spec.authors       = ["Yuichiro Someya"]
+  spec.email         = ["ayemos.y@gmail.com"]
+
+  spec.summary       = %q{AWS Batch DSL and a set of cli tool.}
+  spec.description   = %q{AWS Batch DSL and a set of cli tool.}
+  spec.homepage      = "https://github.com/ayemos/bako"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "awesome_print"
+
+  spec.add_dependency "aws-sdk", "~> 2"
+  spec.add_dependency "thor", "~> 0.19"
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "bako"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/example/hello/hello.rb

@@ -0,0 +1,32 @@
+preproc_jd = job_definition 'preproc-inception' do
+  command ['sleep', '10']
+  type 'container'
+  image 'busybox'
+end
+
+preproc_jobs = []
+
+[*1..2].each do |i|
+  preproc_job = job "preproc-inception-#{i}" do
+    job_definition preproc_jd
+
+    param({
+      foo: 'bar'
+    })
+  end
+
+  preproc_jobs << preproc_job
+end
+
+train_jd = job_definition 'train-inception' do
+  command ['python', '/tensorflow/tensorflow/examples/learn/mnist.py']
+  type 'container'
+  memory 1024
+  vcpus 16
+  image 'tensorflow/tensorflow:latest-devel'
+end
+
+job 'train-inception' do
+  job_definition train_jd
+  #depends_on preproc_jobs
+end

data/exe/bako

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+retried = false
+begin
+  require 'bako'
+  require 'bako/cli'
+rescue LoadError
+  $:.unshift File.expand_path(File.join(__dir__, '..', 'lib'))
+  if retried
+    raise
+  else
+    retried = true
+    retry
+  end
+end
+
+Bako::CLI.start(ARGV)

data/lib/bako.rb

@@ -0,0 +1,17 @@
+require "bako/version"
+require 'bako/common_helper'
+
+module Bako
+  def Bako.logger
+    @logger ||= Logger.new(STDOUT)
+  end
+end
+
+require 'bako/dsl'
+require 'bako/dsl/context'
+require 'bako/dsl/context/job'
+require 'bako/dsl/context/job_definition'
+
+require 'bako/models/job'
+require 'bako/models/job_definition'
+

data/lib/bako/cli.rb

@@ -0,0 +1,14 @@
+require 'thor'
+
+module Bako
+  class CLI < Thor
+    class_option :verbose, aliases: '-v', type: :boolean
+    map '--version' => :print_version
+
+    desc 'run', 'run job'
+    def runjob(path)
+      require 'bako/cli/run'
+      Bako::CLI::Run.new(path, options, self).run
+    end
+  end
+end

data/lib/bako/cli/run.rb

@@ -0,0 +1,38 @@
+module Bako
+  class CLI::Run
+    include Bako::CommonHelper
+
+    def initialize(path, options, thor)
+      @options = options
+      @thor = thor
+      @path = path
+    end
+
+    def run
+      dsl.job_definitions.each do |_, jd_context|
+        jd = Bako::Models::JobDefinition.from_context(jd_context)
+
+        if jd.remote_exists?
+          y_or_n = @thor.ask("JobDefinition #{jd.name} seems to exist on remote. would you like to update it? (y/n)")
+          next unless y_or_n =~ /y/i
+        end
+
+        jd.register
+      end
+
+      jobs_to_be_run.each do |job|
+        Bako::Models::Job.from_context(job).start
+      end
+    end
+
+    private
+
+    def jobs_to_be_run
+      dsl.jobs.values - dsl.jobs.values.map{|j| j.depends_on_b}.flatten
+    end
+
+    def dsl
+      @dsl ||= Bako::DSL.parse(File.read(@path))
+    end
+  end
+end

data/lib/bako/common_helper.rb

@@ -0,0 +1,37 @@
+require 'aws-sdk'
+
+module Bako
+  module CommonHelper
+    DEFAULTS = {
+      vcpus: 1,
+      memory: 128
+    }
+    def batch_client
+      @batch_client ||= Aws::Batch::Client.new
+    end
+    def remote_resource(arn)
+    end
+
+    def remote_job_definition(name_or_arn, status=nil)
+      remote_job_definitions(status).find{|jd|
+        jd.name == name_or_arn || (jd.arn || jd.arn == name_or_arn)
+      }
+    end
+
+    def remote_job_definitions(status=nil)
+      batch_client.describe_job_definitions({
+        status: status
+      }).job_definitions.map do |jd|
+        Bako::Models::JobDefinition.new(
+          jd.job_definition_name,
+          jd.container_properties&.command,
+          jd.container_properties&.image,
+          jd.container_properties&.job_role_arn,
+          jd.type,
+          jd.container_properties&.memory,
+          jd.container_properties&.vcpus
+        )
+      end
+    end
+  end
+end

data/lib/bako/compute_environment.rb

@@ -0,0 +1,3 @@
+module Bako
+
+end

data/lib/bako/dsl.rb

@@ -0,0 +1,8 @@
+class Bako::DSL
+  class << self
+    def parse(dsl)
+      require 'bako/dsl/context'
+      Bako::DSL::Context.eval(dsl)
+    end
+  end # of class methods
+end

data/lib/bako/dsl/context.rb

@@ -0,0 +1,38 @@
+class Bako::DSL::Context
+  attr_reader :jobs, :job_definitions
+
+  def self.eval(dsl)
+    new do
+      eval(dsl)
+    end
+  end
+
+  def initialize(&block)
+    @jobs = {}
+    @job_definitions = {}
+
+    instance_eval(&block)
+  end
+
+  private
+
+  def job(name, &block)
+    name = name.to_s
+
+    if @jobs[name]
+      raise "Job `#{name}` is already defined"
+    end
+
+    @jobs[name] = Bako::DSL::Context::Job.new(name, &block)
+  end
+
+  def job_definition(name, &block)
+    name = name.to_s
+
+    if @job_definitions[name]
+      raise "JobDefinition `#{name}` is already defined"
+    end
+
+    @job_definitions[name] = Bako::DSL::Context::JobDefinition.new(name, &block)
+  end
+end

data/lib/bako/dsl/context/job.rb

@@ -0,0 +1,53 @@
+module Bako
+  class DSL::Context::Job
+    attr_reader :name, :param_b, :job_definition_b, :depends_on_b, :memory_b, :vcpus_b
+
+    def initialize(name, &block)
+      @name = name
+      @param_b = {}
+
+      instance_eval(&block)
+    end
+
+    private
+
+    def job_definition(jd)
+      if jd.nil?
+        raise InvalidArgumentError.new('JobDefinition must be set')
+      end
+
+      @job_definition_b =
+        if jd.is_a?(Bako::DSL::Context::JobDefinition)
+          jd
+        elsif jd.is_a?(String)
+          remote_job_definition(jd)
+        else
+          raise InvalidArgumentError.new('JobDefinition must be String or Bako::DSL::Context::JobDefinition')
+        end
+    end
+
+    def depends_on(jobs)
+      @depends_on_b = jobs&.map do |job|
+        if job.is_a?(Bako::DSL::Context::Job)
+          job
+        elsif job.is_a?(String)
+          remote_job(job)
+        else
+          raise InvalidArgumentError.new('Job must be String or Bako::DSL::Context::Job')
+        end
+      end
+    end
+
+    def param(h)
+      @param_b = h
+    end
+
+    def memory(memory_b)
+      @memory_b = memory_b
+    end
+
+    def vcpus(vcpus_b)
+      @vcpus_b = vcpus_b
+    end
+  end
+end

data/lib/bako/dsl/context/job_definition.rb

@@ -0,0 +1,39 @@
+
+module Bako
+  class DSL::Context::JobDefinition
+    attr_reader :name, :type_b, :image_b, :command_b, :role_arn_b, :memory_b, :vcpus_b
+
+    def initialize(name, &block)
+      @name = name
+      @type_b = 'container'
+
+      instance_eval(&block)
+    end
+
+    private
+
+    def type(job_type)
+      @type_b = job_type
+    end
+
+    def image(image_b)
+      @image_b = image_b
+    end
+
+    def command(command_b)
+      @command_b = command_b
+    end
+
+    def role_arn(role_arn_b)
+      @role_arn_b = role_arn_b
+    end
+
+    def memory(memory_b)
+      @memory_b = memory_b
+    end
+
+    def vcpus(vcpus_b)
+      @vcpus_b = vcpus_b
+    end
+  end
+end

data/lib/bako/error.rb

@@ -0,0 +1,4 @@
+module Hako
+  class InvalidArgumentError < StandardError
+  end
+end

data/lib/bako/models/job.rb

@@ -0,0 +1,57 @@
+require 'yaml'
+require 'pathname'
+
+module Bako
+  module Models
+    class Job
+      include Bako::CommonHelper
+
+      attr_reader :name, :job_definition, :depends_on, :param, :id, :memory, :vcpus
+
+      def self.from_context(context)
+        new(
+          context.name,
+          context.job_definition_b,
+          context.depends_on_b&.map{|job_context|
+            Bako::Models::Job.from_context(job_context)
+          },
+          context.param_b,
+          context.memory_b,
+          context.vcpus_b
+        )
+      end
+
+      def initialize(name, job_definition, depends_on, param, memory, vcpus)
+        @name = name
+        @job_definition = job_definition
+        @depends_on = depends_on
+        @param = param
+        @memory = memory
+        @vcpus = vcpus
+      end
+
+      def start
+        # start 'depends_on' jobs first
+        d_ids = []
+
+        @depends_on&.each do |job|
+          d_ids << job.start.job_id
+        end
+
+        # start job
+        resp = batch_client.submit_job({
+          job_definition: @job_definition.name,
+          job_name: @name,
+          job_queue: 'ayemos-batche-queue-002',
+          depends_on: @depends_on&.map{|j| [[:job_id, j.id]].to_h},
+          parameters: @param
+        })
+
+        @id = resp.job_id
+        Bako.logger.info("Submitted job #{@name} (id: #{@id})")
+
+        resp
+      end
+    end
+  end
+end

data/lib/bako/models/job_definition.rb

@@ -0,0 +1,57 @@
+require 'yaml'
+require 'pathname'
+
+module Bako
+  module Models
+    class JobDefinition
+      include Bako::CommonHelper
+
+      attr_reader :name, :command, :role_arn, :image, :type, :arn, :memory, :vcpus
+
+      def self.from_context(context)
+        new(
+          context.name,
+          context.command_b,
+          context.image_b,
+          context.role_arn_b,
+          context.type_b,
+          context.memory_b,
+          context.vcpus_b
+        )
+      end
+
+      def initialize(name, command, image, role_arn, type, memory, vcpus)
+        @name = name
+        @command = command
+        @image = image
+        @role_arn = role_arn
+        @type = type
+        @memory = memory
+        @vcpus = vcpus
+      end
+
+      def register
+        Bako.logger.info("Registering JobDefinition #{@name}")
+
+        resp = batch_client.register_job_definition({
+          type: @type,
+          container_properties: {
+            command: @command,
+            image: @image,
+            memory: @memory || DEFAULTS[:memory],
+            vcpus: @vcpus || DEFAULTS[:vcpus],
+          },
+          job_definition_name: @name
+        })
+
+        @arn = resp.job_definition_arn
+        @revision = resp.revision
+        Bako.logger.info("Registered JobDefinition #{@name}:#{@revision} (arn: #{@arn})")
+      end
+
+      def remote_exists?
+        !remote_job_definition(@name, 'ACTIVE').nil?
+      end
+    end
+  end
+end

data/lib/bako/version.rb

@@ -0,0 +1,3 @@
+module Bako
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,152 @@
+--- !ruby/object:Gem::Specification
+name: bako
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Yuichiro Someya
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-04-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+description: AWS Batch DSL and a set of cli tool.
+email:
+- ayemos.y@gmail.com
+executables:
+- bako
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- bako.gemspec
+- bin/console
+- bin/setup
+- example/hello/hello.rb
+- exe/bako
+- lib/bako.rb
+- lib/bako/cli.rb
+- lib/bako/cli/run.rb
+- lib/bako/common_helper.rb
+- lib/bako/compute_environment.rb
+- lib/bako/dsl.rb
+- lib/bako/dsl/context.rb
+- lib/bako/dsl/context/job.rb
+- lib/bako/dsl/context/job_definition.rb
+- lib/bako/error.rb
+- lib/bako/models/job.rb
+- lib/bako/models/job_definition.rb
+- lib/bako/version.rb
+homepage: https://github.com/ayemos/bako
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: AWS Batch DSL and a set of cli tool.
+test_files: []