ev_job_gen 0.1.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f81e0ba7d2e6d4f0d67275f9141ef6ccaebadd6c4eb9948172166041ff3ffda1
-  data.tar.gz: a4d81e07a49bed567582cd51045225f79348e88e90dc2f6c5f4ece29b1dfe194
+  metadata.gz: 7d76575d155e4b7b154b3a6c99935b9d95e6dcbc4808850300c66970060c7ac0
+  data.tar.gz: 8052e28d5ed38ca4dc74cf947743c5e0b50cb9a19235d68f97352291b3c0ba91
 SHA512:
-  metadata.gz: ef4cd14d6f5444a9cac80825cb04e9aa26092464f806eba24754cda4194b52d42a3fd9e4d65762f15740b5923065927474a2f6d2e33c36c1a03a4406478efcb3
-  data.tar.gz: 6aad2d6e3e648987cec933daf0ba1ed1a45cb1932c90601b2b9acdfcb7d8ba2cddffae273a12f0f6edc2c1b2466b9dba09202fc2f38f71d12f1b277a902807e1
+  metadata.gz: fcdb61bac8e410866278924228bffee720bfbbb1aad4d6cad6f1417f28f662cf232122deed5ddd3d48034a20dfd4129cf05a78ae1a2e2ace7edf0e8d9b91df15
+  data.tar.gz: b631dadb3640383d4e0c574200d1a7c11e2a333e7458fa5b2de3ab55bfdd803b86f53b2346e7fd75cf51ff4be534321bb5898da9f0ba101c62dcc7b206986583

data/.travis.yml

@@ -3,3 +3,4 @@ language: ruby
 rvm:
   - 2.5.1
 before_install: gem install bundler -v 1.16.2
+script: bundle exec rspec

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ev_job_gen (0.1.1)
+    ev_job_gen (0.5.0)
       dotenv (~> 2.5.0)
 
 GEM
@@ -17,7 +17,7 @@ GEM
     pry-doc (0.13.4)
       pry (~> 0.11)
       yard (~> 0.9.11)
-    rake (10.5.0)
+    rake (13.0.1)
     rspec (3.7.0)
       rspec-core (~> 3.7.0)
       rspec-expectations (~> 3.7.0)
@@ -31,7 +31,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.7.0)
     rspec-support (3.7.1)
-    yard (0.9.15)
+    yard (0.9.20)
 
 PLATFORMS
   ruby
@@ -41,8 +41,8 @@ DEPENDENCIES
   ev_job_gen!
   pry
   pry-doc
-  rake (~> 10.0)
+  rake (~> 13.0)
   rspec (~> 3.7)
 
 BUNDLED WITH
-   1.16.2
+   1.17.2

data/README.md

@@ -25,11 +25,12 @@ Place a yml file in jobs directory
 # Ex. ml-project-1.yml
 namespace: ml-project-1
 image: some-docker-registory/ml-project-1:latest
-command: >
-         script/evaluate | tee /tmp/out.txt;
+command:
+- script/evaluate | tee /tmp/out.txt
 out: /tmp/out.txt
 metrics: /tmp/metrics.json
 commithash: ${COMMITHASH}
+setup_wget: apt install -y wget
 ```
 
 Then execute `ev-job-gen` command.
@@ -74,30 +75,29 @@ spec:
             app: ev-job
             role: job
         spec:
+
           restartPolicy: Never
           containers:
           - name: ev-job
-            image: some-docker-registory/wantedly/ml-project-1:latest
+            image: some-docker-registory/ml-project-1:latest
             imagePullPolicy: Always
             command:
               - "/bin/bash"
               - "-c"
               - >
-                script/evaluate | tee /tmp/out.txt;
-
-                export AWS_ACCESS_KEY_ID=xxx;
-                export AWS_SECRET_ACCESS_KEY=xxx;
-                export AWS_REGION=xxx;
-
-                wget https://github.com/wantedly/ev-cli/releases/download/v1.2.3/ev-v1.2.3-linux-amd64.tar.gz;
-                tar xvzf ev-v1.2.3-linux-amd64.tar.gz;
-                cp linux-amd64/ev ev;
-
+                script/evaluate | tee /tmp/out.txt &&
+                export AWS_ACCESS_KEY_ID=xxx &&
+                export AWS_SECRET_ACCESS_KEY=xxx &&
+                export AWS_REGION=xxx &&
+                apt install -y wget &&
+                wget https://github.com/wantedly/ev-cli/releases/download/v1.2.3/ev-v1.2.3-linux-amd64.tar.gz &&
+                tar xvzf ev-v1.2.3-linux-amd64.tar.gz &&
+                cp linux-amd64/ev ev &&
                 ./ev upload --branch master \
                             --commit ${COMMITHASH} \
                             --out /tmp/out.txt \
                             --metrics /tmp/metrics.json \
-                            --namespace ml-project-1;
+                            --namespace ml-project-1
 ```
 
 ## Development

data/ev_job_gen.gemspec

@@ -25,7 +25,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency             "dotenv", "~> 2.5.0"
   spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.7"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "pry-doc"

data/kubernetes/tmpl/cron_job_template.yml.erb

@@ -14,6 +14,7 @@ spec:
   suspend: false
   successfulJobsHistoryLimit: 10
   failedJobsHistoryLimit: 3
+  startingDeadlineSeconds: 300
   jobTemplate:
     metadata:
       name: ev-job
@@ -30,4 +31,6 @@ spec:
             name: ev-job
             app: ev-job
             role: job
+          annotations:
+            cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
         spec: <%= spec %>

data/kubernetes/tmpl/oneshot_job_template.yml.erb

@@ -9,6 +9,7 @@ metadata:
     app: ev-job
     role: job
 spec:
+  backoffLimit: 0
   template:
     metadata:
       name: ev-job-<%= job.suffix %>

data/kubernetes/tmpl/spec_template.yml.erb

@@ -8,19 +8,17 @@ containers:
     - "/bin/bash"
     - "-c"
     - >
-      <%= job.command.strip %>
-
-      export AWS_ACCESS_KEY_ID=<%= ENV['AWS_ACCESS_KEY_ID'] %>;
-      export AWS_SECRET_ACCESS_KEY=<%= ENV['AWS_SECRET_ACCESS_KEY'] %>;
-      export AWS_REGION=<%= ENV['AWS_REGION'] %>;
-
-      wget https://github.com/wantedly/ev-cli/releases/download/v1.2.3/ev-v1.2.3-linux-amd64.tar.gz;
-      tar xvzf ev-v1.2.3-linux-amd64.tar.gz;
-      cp linux-amd64/ev ev;
-
+      <%= job.command.join(" && ") %> &&
+      export AWS_ACCESS_KEY_ID=<%= ENV['AWS_ACCESS_KEY_ID'] %> &&
+      export AWS_SECRET_ACCESS_KEY=<%= ENV['AWS_SECRET_ACCESS_KEY'] %> &&
+      export AWS_REGION=<%= ENV['AWS_REGION'] %> &&
+      <%= job.setup_wget %> &&
+      wget https://github.com/wantedly/ev-cli/releases/download/v1.2.3/ev-v1.2.3-linux-amd64.tar.gz &&
+      tar xvzf ev-v1.2.3-linux-amd64.tar.gz &&
+      cp linux-amd64/ev ev &&
       ./ev upload --branch master \
                   --commit <%= job.commithash %> \
                   --out <%= job.out %> \
                   --metrics <%= job.metrics %> \
-                  --namespace <%= job.namespace %>;
+                  --namespace <%= job.namespace %>
 <%= config.containers_envFrom(offset: 2) %>

data/lib/ev_job_gen.rb

@@ -3,8 +3,8 @@ require 'dotenv/load'
 require 'ev_job_gen/cli'
 require 'ev_job_gen/config'
 require 'ev_job_gen/job'
-require 'ev_job_gen/yml_renderer'
 require 'ev_job_gen/manifest_gen'
+require 'ev_job_gen/renderer'
 
 module EvJobGen
   class << self

data/lib/ev_job_gen/job.rb

@@ -13,12 +13,22 @@ module EvJobGen
       @out        = h["out"]
       @metrics    = h["metrics"]
       @commithash = h["commithash"]
+      @setup_wget = h["setup_wget"]  # NOTE: this is optional
 
       # NOTE: suffix is used only for oneshot job
       @suffix    = 5.times.map { CHARS.sample }.join
     end
 
-    attr_reader :namespace, :image, :command, :out, :metrics, :commithash, :suffix
+    attr_reader(
+      :namespace,
+      :image,
+      :command,
+      :out,
+      :metrics,
+      :commithash,
+      :setup_wget,
+      :suffix,
+    )
 
   private
 

data/lib/ev_job_gen/manifest_gen.rb

@@ -20,7 +20,7 @@ module EvJobGen
       else
         raise "invalid target: #{target}"
       end
-      @renderer = YmlRenderer.new(
+      @renderer = Renderer.new(
         template:   template,
         specoffset: specoffset,
         jobfile:    jobfile,

data/lib/ev_job_gen/renderer.rb

@@ -0,0 +1,38 @@
+require 'ev_job_gen/renderer/manifest_renderer'
+require 'ev_job_gen/renderer/spec_renderer'
+
+module EvJobGen
+  class Renderer
+    def initialize(template:, specoffset:, jobfile:, configfile:)
+      @template      = template
+      @specoffset    = specoffset
+      @job           = EvJobGen::Job.new(jobfile)
+      @config        = EvJobGen::Config.new(configfile)
+    end
+
+    def render
+      spec = SpecRenderer.new(
+        job:    @job,
+        config: @config,
+      ).render
+
+      r = ManifestRenderer.new(
+        template: @template,
+        job:      @job,
+        spec:     offset(spec, off: @specoffset + 2)
+      ).render
+
+      normalize(r)
+    end
+
+  private
+
+    def normalize(text)
+      text.split("\n").map { |l| l.rstrip }.map { |l| "#{l}\n" }.join
+    end
+
+    def offset(text, off:)
+      "\n" + text.split("\n").map { |l| "#{' ' * off}#{l}\n" }.join
+    end
+  end
+end

data/lib/ev_job_gen/renderer/manifest_renderer.rb

@@ -0,0 +1,18 @@
+require 'erb'
+
+module EvJobGen
+  class Renderer
+    class ManifestRenderer
+      def initialize(template:, job:, spec:)
+        @renderer = ERB.new(File.read(template))
+        @job      = job
+        @spec     = spec
+      end
+      attr_reader :job, :spec
+
+      def render
+        @renderer.result(binding)  # Only use `job` and `spec`
+      end
+    end
+  end
+end

data/lib/ev_job_gen/renderer/spec_renderer.rb

@@ -0,0 +1,27 @@
+require 'erb'
+
+module EvJobGen
+  class Renderer
+    class SpecRenderer
+      class << self
+        def spec_template_path
+          File.expand_path(
+            File.join('kubernetes', 'tmpl', 'spec_template.yml.erb'),
+            EvJobGen.root,
+          )
+        end
+      end
+
+      def initialize(job:, config:)
+        @job        = job
+        @config     = config
+        @renderer   = ERB.new(File.read(SpecRenderer.spec_template_path))
+      end
+      attr_reader :job, :config
+
+      def render
+        @renderer.result(binding)  # Only use `job` and `config`
+      end
+    end
+  end
+end

data/lib/ev_job_gen/version.rb

@@ -1,3 +1,3 @@
 module EvJobGen
-  VERSION = "0.1.1"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ev_job_gen
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Nao Minami
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2018-08-06 00:00:00.000000000 Z
+date: 2020-11-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -125,13 +125,15 @@ files:
 - lib/ev_job_gen/config.rb
 - lib/ev_job_gen/job.rb
 - lib/ev_job_gen/manifest_gen.rb
+- lib/ev_job_gen/renderer.rb
+- lib/ev_job_gen/renderer/manifest_renderer.rb
+- lib/ev_job_gen/renderer/spec_renderer.rb
 - lib/ev_job_gen/version.rb
-- lib/ev_job_gen/yml_renderer.rb
 homepage: https://github.com/wantedly/ev-job-gen
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -146,9 +148,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.0.3
+signing_key:
 specification_version: 4
 summary: Generator of kubernetes manifest file for evaluation job.
 test_files: []

data/lib/ev_job_gen/yml_renderer.rb

@@ -1,38 +0,0 @@
-require 'erb'
-require 'fileutils'
-
-module EvJobGen
-  class YmlRenderer
-    class << self
-      def spec_template_path
-        File.expand_path(
-          File.join('kubernetes', 'tmpl', 'spec_template.yml.erb'),
-          EvJobGen.root,
-        )
-      end
-    end
-
-    def initialize(template:, specoffset:, jobfile:, configfile:)
-      @spec_renderer = ERB.new(File.read(YmlRenderer.spec_template_path))
-      @specoffset    = specoffset
-      @renderer      = ERB.new(File.read(template))
-      @job           = EvJobGen::Job.new(jobfile)
-      @config        = EvJobGen::Config.new(configfile)
-    end
-
-    attr_reader :job, :config
-
-    def render
-      spec = render_spec
-      @renderer.result(binding)
-    end
-
-  private
-
-    def render_spec
-      r = @spec_renderer.result(binding)
-      offset = @specoffset + 2
-      "\n" + r.split("\n").map { |l| "#{' ' * offset}#{l}\n" }.join
-    end
-  end
-end