kuroko2-updater 0.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cd5411880d8d46b6f01127b10f15c1b7949e4eb4f024d5481cab786c03912d66
+  data.tar.gz: 1689cfe567e18c01524d5b13096a36f26b702d83a1c7cd62b1ac843ab5c046b3
+SHA512:
+  metadata.gz: dc67e62689aad9f7c2f6f5d35eca57cb02040967dce1c6e730a6bed5b82db3c123a2cb7d1794024e7b092ffd806744aca8f6e29eaf2e0f772347f4a2a98c3a25
+  data.tar.gz: 4f4508b81ed8637dea3dfc6e37e9662f31840fb3d86fd0a6de2d6a7fa0212664b009b622c5414bfcadf9e0536741d0412b2586e45856abce73cfa36e2ff222c5

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+# Rubocop remote configs
+.rubocop-*

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,2 @@
+inherit_from:
+  - https://raw.githubusercontent.com/cookpad/global-style-guides/master/.rubocop.yml

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.6.1
+before_install: gem install bundler -v 2.0.1

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in kuroko2-updater.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,88 @@
+PATH
+  remote: .
+  specs:
+    kuroko2-updater (0.1.2)
+      activesupport
+      faraday (>= 0.15)
+      faraday_middleware (>= 0.13)
+      whenever (~> 0.11)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.0.2.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.2)
+    ast (2.4.0)
+    chronic (0.10.2)
+    concurrent-ruby (1.1.6)
+    diff-lcs (1.3)
+    faraday (1.0.1)
+      multipart-post (>= 1.2, < 3)
+    faraday_middleware (1.0.0)
+      faraday (~> 1.0)
+    i18n (1.8.2)
+      concurrent-ruby (~> 1.0)
+    jaro_winkler (1.5.4)
+    minitest (5.14.0)
+    multipart-post (2.1.1)
+    parallel (1.19.1)
+    parser (2.7.1.2)
+      ast (~> 2.4.0)
+    rack (2.2.2)
+    rainbow (3.0.0)
+    rake (13.0.1)
+    rexml (3.2.4)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.1)
+      rspec-support (~> 3.9.1)
+    rspec-expectations (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.2)
+    rubocop (0.82.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.7.0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      rexml
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-performance (1.5.2)
+      rubocop (>= 0.71.0)
+    rubocop-rails (2.5.2)
+      activesupport
+      rack (>= 1.1)
+      rubocop (>= 0.72.0)
+    ruby-progressbar (1.10.1)
+    thread_safe (0.3.6)
+    tzinfo (1.2.7)
+      thread_safe (~> 0.1)
+    unicode-display_width (1.7.0)
+    whenever (0.11.0)
+      chronic (>= 0.6.3)
+    zeitwerk (2.3.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  kuroko2-updater!
+  rake (~> 13.0)
+  rspec (~> 3.0)
+  rubocop (~> 0.71)
+  rubocop-performance
+  rubocop-rails
+
+BUNDLED WITH
+   2.1.4

data/README.md

@@ -0,0 +1,88 @@
+# Kuroko2 Updater 
+
+## Background
+
+A tool that uses a [whenever](https://github.com/javan/whenever) style config
+file to configure [kuroko2](https://github.com/cookpad/kuroko2)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'kuroko2-updater'
+```
+
+And then execute:
+
+    $ bundle
+
+## Status
+
+Experimental, not intended for general use, no plans for ongoing maintenance.
+
+## Usage
+
+Some modifications must be made to the schedule.rb file to support using
+kuroko2, rather than cron, here is an example of the additions required:
+
+
+```ruby
+set :job_template, <<~TEMPLATE
+  queue: hako-executor
+  docker_application: hello-world-batch
+  env: :environment_variable=:environment
+  hako_oneshot: :job
+TEMPLATE
+
+job_type :rake, "bundle exec rake :task"
+
+# These tags scope the job definitions that will be managed with this config.
+set :kuroko2_tags, %w(hello-world production scheduled)
+
+# These are kuroku2 user ids for the Admin user(s) of the jobs managed
+# by this tool. See the /users page to find the id of the
+# user(s) you want to have this responsibility.
+set :kuroko2_users, [1]
+
+# This description is added to each job definition created in kuroko2 to
+# warn people not to manually edit the jobs managed by this file!
+set :kuroko2_job_description, <<~DESC
+  This job is managed by an automation tool.
+
+  **Warning DO NOT EDIT**
+
+  Please do not edit this job definition directly
+  but instead make a pull request to https://github.com/cookpad/hello-world/blob/master/config/schedule.rb
+DESC
+
+set :kuroko2_slack_channel, "#hello-world-alerts"
+```
+
+As part of the deployment process run the `kuroko2-updater` command.
+
+By default this reads `config/schedule.rb` but another file can be passed as an argument.
+
+## Config
+
+This config is **required** and must be included in the environment for this
+tool to run without error!
+
+| Var | notes | e.g |
+|-----|-------|-----|
+| `KUROKO2_API_URL` | The URL of the kuroko2 API | `https://kuroko2.example.com/v1` |
+| `KUROKO2_API_USER` | The username for kuroku2 API auth | `hello-world` |
+| `KUROKO2_API_KEY` | The password/key for kuroku2 API auth | `secret-password` |
+
+## Warning
+
+This gem is currently only intended to be used internally at Cookpad. And we cannot
+really recommend it's use as we created it to ease some migration issues in a
+specific project.
+
+This tool heavily monkey patches and exposes some internal details of whenever
+in a way that I am sure is not supported by the author of that tool, so it
+may well break unexpectedly if you update whenever!
+
+We intend to improve this tool, or create other tooling to enable this workflow
+more generically, at some point in the future.

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+require "rubocop/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+RuboCop::RakeTask.new
+
+task default: %i(spec rubocop)

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "kuroko2/updater"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/kuroko2-updater

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require "kuroko2/updater"
+
+schedule_config = ARGV[0] || "config/schedule.rb"
+
+Kuroko2::Updater.new(
+  job_list: Whenever::JobList.new(file: schedule_config)
+).run

data/kuroko2-updater.gemspec

@@ -0,0 +1,32 @@
+lib = File.expand_path("lib", __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "kuroko2/updater/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "kuroko2-updater"
+  spec.version       = Kuroko2::Updater::VERSION
+  spec.authors       = ["Ed Robinson"]
+  spec.email         = ["edward-robinson@cookpad.com"]
+  spec.summary       = "A tool to manage kuroko2 job-definitions based on a schedule.rb file"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activesupport"
+  spec.add_dependency "faraday", ">= 0.15"
+  spec.add_dependency "faraday_middleware", ">= 0.13"
+  spec.add_dependency "whenever", "~> 0.11"
+
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rubocop", "~> 0.71"
+  spec.add_development_dependency "rubocop-performance"
+  spec.add_development_dependency "rubocop-rails"
+end

data/lib/kuroko2/updater.rb

@@ -0,0 +1,137 @@
+require "kuroko2/updater/version"
+require "kuroko2/updater/whenever"
+require "kuroko2/updater/client/definition"
+require "logger"
+
+module Kuroko2
+  class Updater
+    class ValidationError < RuntimeError; end
+
+    def initialize(job_list:, client: Kuroko2::Updater::Client::Definition.new)
+      @client = client
+      @job_list = job_list
+    end
+
+    def run
+      validate_job_list
+
+      to_delete.each { |job| client.delete(job) }
+      to_create.each { |job| client.create(job) }
+      to_update.each { |job| client.update(job) }
+    end
+
+    private
+
+      def validate_job_list
+        fail ValidationError, ":kuroko2_tags was not set set in the config file" unless tags.any?
+        fail ValidationError, ":kuroko2_users was not set in the config file" unless users.any?
+        fail ValidationError, "#{duplicated_jobs.join(',')} was specified more than once" if duplicated_jobs.any?
+      end
+
+      def duplicated_jobs
+        names = desired_jobs.map { |job| job[:name] }
+        names.select { |name| names.count(name) > 1 }.uniq
+      end
+
+      def to_delete
+        existing_jobs.reject do |existing_job|
+          desired_jobs.any? do |desired_job|
+            desired_job[:name] == existing_job[:name]
+          end
+        end
+      end
+
+      def to_create
+        desired_jobs.reject do |desired_job|
+          existing_jobs.any? do |existing_job|
+            desired_job[:name] == existing_job[:name]
+          end
+        end
+      end
+
+      def to_update
+        (desired_jobs - to_create).map do |job|
+          job[:id] = id_for(job)
+          job
+        end
+      end
+
+      def id_for(job)
+        existing_jobs.find { |existing_job| existing_job[:name] == job[:name] }[:id]
+      end
+
+      def existing_jobs
+        @_existing_jobs ||= client.list(tags)
+      end
+
+      def desired_jobs
+        @_desired_jobs ||= job_list.time_and_jobs.flat_map do |time, jobs|
+          jobs.map do |job|
+            {
+              name: job.task,
+              script: job.output,
+              description: job_description,
+              cron: cron(time, job),
+              slack_channel: slack_channel,
+              user_id: users,
+              tags: tags,
+              hipchat_notify_finished: notify_finished,
+              notify_cancellation: notify_cancellation
+            }
+          end
+        end
+      end
+
+      def slack_channel
+        job_list.kuroko2_slack_channel
+      rescue NoMethodError
+        nil
+      end
+
+      def job_description
+        job_list.kuroko2_job_description
+      rescue NoMethodError
+        <<~DESC
+          This job is managed by an automation tool.
+
+          **Warning DO NOT EDIT**
+
+          Please do not edit this job definition directly.
+        DESC
+      end
+
+      def notify_finished
+        job_list.kuroko2_notify_finished
+      rescue NoMethodError
+        false
+      end
+
+      def notify_cancellation
+        job_list.kuroko2_notify_cancellation
+      rescue NoMethodError
+        true
+      end
+
+      def tags
+        job_list.kuroko2_tags
+      rescue NoMethodError
+        []
+      end
+
+      def users
+        job_list.kuroko2_users
+      rescue NoMethodError
+        []
+      end
+
+      def cron(times, job)
+        ::Whenever::Output::Cron.enumerate(times).flat_map do |time|
+          ::Whenever::Output::Cron.enumerate(job.at).map do |at|
+            ::Whenever::Output::Cron.new(time, nil, at).time_in_cron_syntax
+          end
+        end
+      end
+
+      attr_reader :job_list, :client
+  end
+end

data/lib/kuroko2/updater/client/definition.rb

@@ -0,0 +1,70 @@
+require "faraday"
+require "faraday_middleware"
+require "active_support/core_ext/hash"
+
+module Kuroko2
+  class Updater
+    class Client
+      class Error < RuntimeError; end
+      class Definition
+        PATH = "definitions".freeze
+
+        def list(tags = nil)
+          response = client.get(PATH) do |req|
+            req.params[:tags] = tags if tags
+          end
+          handle_errors(response)
+          response.body.map(&:deep_symbolize_keys)
+        end
+
+        def create(params)
+          log.info "creating: #{params[:name]}"
+          response = client.post(PATH) do |req|
+            req.body = params
+          end
+          handle_errors(response)
+          response.body.deep_symbolize_keys
+        end
+
+        def update(params)
+          params = params.dup
+          id = params.delete(:id)
+          log.info "updating: #{id} #{params[:name]}"
+          response = client.put("#{PATH}/#{id}") do |req|
+            req.body = params
+          end
+          handle_errors(response)
+          true
+        end
+
+        def delete(params)
+          params = params.dup
+          id = params.delete(:id)
+          log.info "deleting: #{id} #{params[:name]}"
+          response = client.delete("#{PATH}/#{id}")
+          handle_errors(response)
+          true
+        end
+
+        private
+
+          def handle_errors(response)
+            fail Kuroko2::Updater::Client::Error, response.body["message"] unless response.success?
+          end
+
+          def log
+            Logger.new(STDERR)
+          end
+
+          def client
+            @_client ||= Faraday.new(url: ENV.fetch("KUROKO2_API_URL")) do |config|
+              config.basic_auth(ENV.fetch("KUROKO2_API_USER"), ENV.fetch("KUROKO2_API_KEY"))
+              config.use FaradayMiddleware::EncodeJson
+              config.use FaradayMiddleware::ParseJson
+              config.adapter Faraday.default_adapter
+            end
+          end
+      end
+    end
+  end
+end

data/lib/kuroko2/updater/version.rb

@@ -0,0 +1,5 @@
+module Kuroko2
+  class Updater
+    VERSION = "0.1.2".freeze
+  end
+end

data/lib/kuroko2/updater/whenever.rb

@@ -0,0 +1,37 @@
+require "whenever"
+
+module Kuroko2
+  class Updater
+    module Whenever
+      class ::Whenever::Job
+        def task
+          @options[:task]
+        end
+
+        protected
+
+          # patch whenever not to chomp newlines, as kuroko2 scripts need them
+          def process_template(template, options)
+            template.gsub(/:\w+/) do |key|
+              before_and_after = [$`[-1..-1], $'[0..0]]
+              option = options[key.sub(":", "").to_sym] || key
+
+              if before_and_after.all? { |c| c == "'" }
+                escape_single_quotes(option)
+              elsif before_and_after.all? { |c| c == '"' }
+                escape_double_quotes(option)
+              else
+                option
+              end
+            end.strip
+          end
+      end
+
+      class ::Whenever::JobList
+        def time_and_jobs
+          @jobs.values.reduce({}, :merge)
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,199 @@
+--- !ruby/object:Gem::Specification
+name: kuroko2-updater
+version: !ruby/object:Gem::Version
+  version: 0.1.2
+platform: ruby
+authors:
+- Ed Robinson
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2020-05-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.15'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.15'
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.13'
+- !ruby/object:Gem::Dependency
+  name: whenever
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.71'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.71'
+- !ruby/object:Gem::Dependency
+  name: rubocop-performance
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- edward-robinson@cookpad.com
+executables:
+- kuroko2-updater
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/kuroko2-updater
+- kuroko2-updater.gemspec
+- lib/kuroko2/updater.rb
+- lib/kuroko2/updater/client/definition.rb
+- lib/kuroko2/updater/version.rb
+- lib/kuroko2/updater/whenever.rb
+homepage: 
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: A tool to manage kuroko2 job-definitions based on a schedule.rb file
+test_files: []