app-rb 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 23df7f2417cf94b6957cde15260ae950348d8861
-  data.tar.gz: 10198299029a90bada242c624e4bfc59bf3c3e1b
+  metadata.gz: b45f2f46e1dcd69be448e51337faaded1bd09579
+  data.tar.gz: 29460c4f73febf5601a536e76ffe0af0b9ed1c4b
 SHA512:
-  metadata.gz: 88b81b190edd57ee17b485e967303421e777f13fa39b22d4d27eed241e86d344ad122abe6db9d1371c8b0f5647ffa87f03550a21837b6ffb3c964bd77f1ba96e
-  data.tar.gz: 780f0b901f9ed60f2194855617ec896cd986d50a9b7eb2577b93e598b538f99846365c5bb51be38f7267d01b6a6a1475a9b3425344a250ede0acb39f49837106
+  metadata.gz: 28186b82f29304966a4faf69fbd533fe5272142b2ca14f45d188b730cab30b340a2fa3debd4a77f46b6b2c4807a52a0315d5975f5c9fba434a7e066850c5bc0c
+  data.tar.gz: 7c56418bf676f3f7dd4ebe60c3c919e515d3b6cf7bfad7d610527e67afeed04909937d52bde1eee24acebda48adc75545e9231da583e7be0fd839f9d595e422c

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## App-rb 0.2.0 (March 25, 2017) ##
+
+* Add `opts` stanza for `pre_deploy` and `deploy` sections.
+* Remember work nodes.
+* Add `restart` and `clean` cli commands.
+* Add backgound jobs.
+* Add `version` config param.
+* Extract all code to classes.
+
 ## App-rb 0.1.0 (March 25, 2017) ##
 
 * First public version.

data/README.md

@@ -2,6 +2,9 @@
 
 Powerfull docker apps deployer.
 
+[![Build Status](https://img.shields.io/travis/uchiru/app-rb/master.svg)]()
+[![Gem Version](https://img.shields.io/gem/v/app-rb.svg)]()
+
 ## Installation
 
 Install last gem version as:

data/exe/app-rb

@@ -1,313 +1,5 @@
 #!/usr/bin/env ruby
-require 'json'
-require 'yaml'
-require 'pp'
-require 'open3'
-Thread.abort_on_exception = true
-
-def yellow(txt); "\e[0;33m#{txt}\e[0m"; end
-def red(txt);    "\e[0;31m#{txt}\e[0m"; end
-def green(txt);  "\e[0;32m#{txt}\e[0m"; end
-def blue(txt);   "\e[0;34m#{txt}\e[0m"; end
-
-def do_it(cmd)
-  puts "[exec] #{cmd}"
-  system(cmd)
-  unless $?.success?
-    puts red("FATAL :(")
-    exit 
-  end
-end
-
-def just_cmd(cmd, skip_exit_status = false)
-  puts "[exec] #{cmd}"
-  output = `#{cmd}`
-  if $?.success? || skip_exit_status
-    output.strip
-  else
-    puts red(output)
-    puts red("FATAL :(")
-    exit
-  end
-end
-
-if ARGV.count < 2
-  puts "Just deploy your apps with docker and consul. Nothing else."
-  puts ""
-  puts "  #{$0} <yml> <command>"
-  puts ""
-  puts "  deploy [hash]      - deploy new version of app"
-  puts "  status             - status of app"
-  puts "  stop               - stop app"
-  exit
-end
-
-CONFIG = YAML.load(File.read(ARGV[0]))
-COMMAND = ARGV[1]
-
-MIN_PORT = 10_000
-MAX_PORT = 50_000
-Node = Struct.new(:name, :ip, :roles)
-NODES = JSON.load(just_cmd("curl -s #{CONFIG["consul"]}/v1/catalog/nodes")).sort_by { |n| 
-  n["Node"]
-}.map { |n|
-  Node.new(n["Node"], n["Address"], (n["Meta"] || {})["roles"].to_s.split(",").reject{ |s| s.to_s.empty? })
-}
-
-def nodes(constraint)
-  constraint ||= {}
-  out = NODES
-  if constraint["role"]
-    out = out.select { |n| n.roles.index(constraint["role"]) }
-  end
-  if constraint["name"]
-    out = out.select { |n| n.name == constraint["name"] }
-  end
-  out
-end
-
-def node(constraint)
-  nodes(constraint).sample
-end
-
-
-def deploy(target = nil)
-  build_node = node(CONFIG["image"]["constraint"])
-  puts "build_node=#{build_node.to_h.inspect}"
-
-  puts blue("+++ CLONE or UPDATE repository")
-  do_it "ssh #{CONFIG["user"]}@#{build_node.ip} bash <<EOF
-    set -e
-    tmpfile=\\$(mktemp /tmp/git-ssh-#{CONFIG["app"]}.XXXXXX)
-    echo '#!/bin/sh' > \\$tmpfile
-    echo 'exec /usr/bin/ssh -o StrictHostKeyChecking=no -i #{CONFIG["image"]["key"]} \"\\$@\"' >> \\$tmpfile
-    chmod +x \\$tmpfile
-    export GIT_SSH=\\$tmpfile
-    if [ -d #{CONFIG["app"]}-cache ]; then
-      echo update cache...
-      cd #{CONFIG["app"]}-cache
-      git checkout . && git clean -dfx && git checkout master && git pull
-      git branch | grep -v master | xargs -r git branch -D
-    else
-      echo clone...
-      git clone git@github.com:#{CONFIG["image"]["repo"]} #{CONFIG["app"]}-cache && cd #{CONFIG["app"]}-cache
-    fi
-    git checkout #{target || CONFIG["image"]["target"]}
-    rm \\$tmpfile\nEOF"
-
-  puts blue("+++ calculate HASH and VERSION")
-  hash = ARGV[2] || `ssh #{CONFIG["user"]}@#{build_node.ip} 'cd #{CONFIG["app"]}-cache && git rev-parse HEAD'`.strip
-  puts "hash: #{hash}"
-
-  o = JSON.load(`curl -s https://#{CONFIG["registry"]}/v2/#{CONFIG["app"]}/tags/list`)
-  tags = o.is_a?(Hash) && o["errors"] ? [] : o["tags"]
-  puts "tags: #{JSON.dump(tags)}"
-
-  unless tags.index(hash)
-    puts blue("+++ BUILD image")
-    do_it "ssh #{CONFIG["user"]}@#{build_node.ip} bash <<EOF
-      set -e
-      cd #{CONFIG["app"]}-cache
-      #{(CONFIG["image"]["pre_build"] || []).join("\n")}
-      docker build -t #{CONFIG["registry"]}/#{CONFIG["app"]}:#{hash} .
-      docker push #{CONFIG["registry"]}/#{CONFIG["app"]}:#{hash}
-      echo Done.\nEOF"
-  end
-  vv = Time.now.to_i
-  new_service = "#{CONFIG["app"]}-#{vv}"
-
-  (CONFIG["deploy"]["pre"] || []).each_with_index do |section, index|
-    puts blue("+++ PRE: #{section.inspect}")
-    n = node(section["constraint"] || CONFIG["deploy"]["constraint"])
-    puts "node=#{n.inspect}"
-    do_it "ssh #{CONFIG["user"]}@#{n.ip} docker run " +
-      "--label app=#{CONFIG["app"]} " +
-      "--label service=#{new_service} " +
-      "--name #{CONFIG["app"]}-pre-#{vv}-#{index} " +
-      "#{(CONFIG["env"] || {}).map { |k, v| "-e #{k}='#{v}'" }.join(" ")} " +
-      "#{CONFIG["registry"]}/#{CONFIG["app"]}:#{hash} #{section["cmd"]}"
-    do_it "ssh #{CONFIG["user"]}@#{n.ip} docker rm #{CONFIG["app"]}-pre-#{vv}-#{index}"
-  end
-
-  deploy_nodes = nodes(CONFIG["deploy"]["constraint"])
-
-  puts blue("+++ PULL")
-  pull_threads = []
-  deploy_nodes.each do |node|
-    pull_threads << Thread.new do
-      host = "#{CONFIG["user"]}@#{node.ip}"
-      Open3.popen2e("ssh #{host} docker pull #{CONFIG["registry"]}/#{CONFIG["app"]}:#{hash}") { |i,o,w|
-        while line = o.gets do 
-          puts "[#{node.name}] " + line
-        end
-        raise "FATAL" unless w.value.success?
-      }
-    end
-  end
-  pull_threads.each(&:join)
-
-  plan = {}
-  if CONFIG["deploy"]["per"]
-    amount = CONFIG["deploy"]["per"]*deploy_nodes.count
-  else
-    amount = CONFIG["deploy"]["amount"]
-  end
-
-  amount.times do |index|
-    ip = deploy_nodes[index % deploy_nodes.length].ip
-    plan[ip] ||= []
-    plan[ip].push(index)
-  end
-
-  puts blue("+++ DEPLOY")
-  deploy_threads = []
-  deploy_nodes.each do |node|
-    deploy_threads << Thread.new do
-      host = "#{CONFIG["user"]}@#{node.ip}"
-
-      puts "[#{node.name}] run"
-      (plan[node.ip] || []).each do |index|
-        port = nil
-        10.times do
-          a = MIN_PORT + rand(MAX_PORT - MIN_PORT)
-          if just_cmd("ssh #{host} ss -ln src :#{a} | fgrep -c ':#{a}'", true) == "0"
-            port = a
-            break
-          end
-        end
-        raise "Dont find free port :-((" unless port 
-        puts "[#{node.name}] port=#{port}"
-
-        do_it("ssh #{host} docker run -d " + 
-              "--label app=#{CONFIG["app"]} " +
-              "--label service=#{new_service} " +
-              "--name=#{CONFIG["app"]}-#{vv}-#{index} " +
-              (CONFIG["env"] || {}).map { |k, v| "-e #{k}='#{v}'" }.join(" ") + " " +
-              "--restart unless-stopped " +
-              "-p #{node.ip}:#{port}:#{CONFIG["deploy"]["port"]} " +
-              "#{CONFIG["registry"]}/#{CONFIG["app"]}:#{hash} " +
-              "#{CONFIG["deploy"]["cmd"]}")
-
-        do_it %(curl -s -X PUT #{node.ip}:8500/v1/agent/service/register -d'{
-          "Id": "#{CONFIG["app"]}-#{vv}-#{index}",
-          "Name": "#{new_service}",
-          "Port": #{port},
-          "Check": {
-            "DeregisterCriticalServiceAfter": "20m",
-            "Interval": "7s",
-            "HTTP": "http://#{node.ip}:#{port}#{CONFIG["deploy"]["check_url"] || "/"}"
-          }
-        }')
-      end
-
-      puts blue("+++ CONSUL wait for #{node.name}")
-      loop do
-        statuses = JSON.load(just_cmd("curl -s #{node.ip}:8500/v1/health/service/#{CONFIG["app"]}-#{vv}")).select { |s|
-          s["Node"]["Address"] == node.ip
-        }.flat_map { |s| s["Checks"] }.map { |c| c["Status"] }
-        puts "#{node.name} => #{statuses.inspect}"
-        break if statuses.uniq == ["passing"] or (plan[node.ip] || []).empty?
-        sleep 3
-      end
-    end
-  end
-  deploy_threads.each(&:join)
-
-  current_service = just_cmd("curl #{CONFIG["consul"]}/v1/kv/apps/#{CONFIG["app"]}/service?raw")
-  current_hash = just_cmd("curl #{CONFIG["consul"]}/v1/kv/apps/#{CONFIG["app"]}/hash?raw")
-  puts "CURRENT_SERVICE=#{current_service}"
-  puts "CURRENT_HASH=#{current_hash}"
-  puts "NEW_SERVICE=#{new_service}"
-  do_it "curl -X PUT #{CONFIG["consul"]}/v1/kv/apps/#{CONFIG["app"]}/service -d#{new_service}"
-  do_it "curl -X PUT #{CONFIG["consul"]}/v1/kv/apps/#{CONFIG["app"]}/hash -d#{hash}"
-  puts "\n\n" + green(">>>>>>>>>>>>>>>  BLUE/GREEN switch <<<<<<<<<<<<<<<")
-  sleep 3
-
-  puts blue("+++ STOP OLD containers")
-  JSON.load(just_cmd("curl -s #{CONFIG["consul"]}/v1/catalog/services")).keys.select { |service|
-    service != new_service && service =~ /^#{CONFIG["app"]}-\d+$/
-  }.each do |service|
-    JSON.load(just_cmd("curl -s #{CONFIG["consul"]}/v1/health/service/#{service}")).each do |s|
-      do_it %(curl -s -X DELETE #{s["Node"]["Address"]}:8500/v1/agent/service/deregister/#{s["Service"]["ID"]})
-    end
-  end
-
-  NODES.each do |n|
-    keep_ids = just_cmd("ssh #{CONFIG["user"]}@#{n.ip} docker ps -q -f label=app=#{CONFIG["app"]} -f label=service=#{new_service}").split("\n")
-    all_ids = just_cmd("ssh #{CONFIG["user"]}@#{n.ip} docker ps -q -f label=app=#{CONFIG["app"]}").split("\n")
-    if (all_ids - keep_ids).length > 0
-      do_it("ssh #{CONFIG["user"]}@#{n.ip} docker stop #{(all_ids - keep_ids).join(" ")}")
-      do_it("ssh #{CONFIG["user"]}@#{n.ip} docker rm #{(all_ids - keep_ids).join(" ")}")
-    end
-  end
-
-  puts blue("+++ CLEAN REGISTRY")
-  (tags - [hash, current_hash]).each do |hash|
-    digest = just_cmd("curl -s --head -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' https://#{CONFIG["registry"]}/v2/#{CONFIG["app"]}/manifests/#{hash} | grep Docker-Content-Digest | cut -d' ' -f2")
-    puts "digest = #{digest}"
-    system "curl -X DELETE https://#{CONFIG["registry"]}/v2/#{CONFIG["app"]}/manifests/#{digest}"
-  end
-
-  puts green("Done.")
-  if current_hash != "" && !target
-    puts "to rollback execute: #{$0} #{ARGV[0]} deploy #{current_hash}"
-  end
-end
-
-
-def print_status
-  max_name_len = NODES.map { |n| n.name.length }.max
-  max_ip_len = NODES.map { |n| n.ip.length }.max
-  max_roles_len = NODES.map { |n| n.roles.inspect.length }.max
-  current_service = just_cmd("curl -s #{CONFIG["consul"]}/v1/kv/apps/#{CONFIG["app"]}/service?raw")
-  current_hash = just_cmd("curl -s #{CONFIG["consul"]}/v1/kv/apps/#{CONFIG["app"]}/hash?raw")
-  current_dockers = NODES.map { |n| 
-    just_cmd("ssh #{CONFIG["user"]}@#{n.ip} 'docker ps -q -f label=app=#{CONFIG["app"]} -f label=service=#{current_service} | wc -l'").to_i
-  }
-  dockers = NODES.map { |n| 
-    just_cmd("ssh #{CONFIG["user"]}@#{n.ip} 'docker ps -q -f label=app=#{CONFIG["app"]} | wc -l'").to_i
-  }
-  puts ""
-  puts green("App:     ") + CONFIG["app"]
-  puts green("Service: ") + current_service
-  puts green("Hash:    ") + current_hash
-  NODES.each_with_index do |n, i|
-    puts(
-      " "*5 + n.name.rjust(max_name_len) + 
-      " "*2 + n.ip.ljust(max_ip_len) + 
-      " "*2 + n.roles.inspect.ljust(max_roles_len) +
-      " "*2 + green(current_dockers[i]) + " / " + (dockers[i] - current_dockers[i] == 0 ? "0" : red(dockers[i] - current_dockers[i]))
-    )
-  end
-end
-
-
-def do_stop
-  current_service = just_cmd("curl -s #{CONFIG["consul"]}/v1/kv/apps/#{CONFIG["app"]}/service?raw")
-  if current_service != ""
-    JSON.load(just_cmd("curl -s #{CONFIG["consul"]}/v1/health/service/#{current_service}")).each do |s|
-      do_it %(curl -s -X DELETE #{s["Node"]["Address"]}:8500/v1/agent/service/deregister/#{s["Service"]["ID"]})
-    end
-  end
-  NODES.map { |n| 
-    ids = just_cmd("ssh #{CONFIG["user"]}@#{n.ip} docker ps -q -f label=app=#{CONFIG["app"]}").gsub("\n", " ")
-    if ids != ""
-      do_it "ssh #{CONFIG["user"]}@#{n.ip} docker stop #{ids}"
-      do_it "ssh #{CONFIG["user"]}@#{n.ip} docker rm #{ids}"
-    end
-  }
-  do_it "curl -X DELETE #{CONFIG["consul"]}/v1/kv/apps/#{CONFIG["app"]}?recurse"
-  puts ""
-end
-
-
-if COMMAND == "deploy" || COMMAND == "d"
-  deploy(ARGV[2])
-elsif COMMAND == "status" || COMMAND == "s"
-  print_status
-elsif COMMAND == "stop"
-  do_stop
-else
-  puts "FATAL: unknown command '#{COMMAND}'"
-end
+require "bundler/setup"
+require "app-rb"
 
+AppRb::Cli.new(ARGV).run

data/lib/app-rb/cli.rb

@@ -1,4 +1,52 @@
+require 'yaml'
+
 module AppRb
   class Cli
+    def initialize(args)
+      @args = args
+      Thread.abort_on_exception = true
+    end
+
+    def run
+      if @args.count < 2
+        usage
+        exit
+      end
+      config = Config.new(YAML.load(File.read(@args[0])))
+      command = @args[1]
+
+      if AppRb::Util.compare_versions(config.tool_version, AppRb::VERSION) > 0
+        puts "FATAL: need at least '#{config.tool_version}' tool version but current version is '#{AppRb::VERSION}'"
+        exit -1
+      end
+
+      if command == "deploy" || command == "d"
+        Command.new(config).deploy(@args[2])
+      elsif command == "status" || command == "s"
+        Command.new(config).status
+      elsif command == "restart"
+        Command.new(config).restart
+      elsif command == "clean"
+        Command.new(config).clean
+      elsif command == "stop"
+        Command.new(config).stop
+      else
+        puts "FATAL: unknown command '#{command}'"
+        exit -1
+      end
+    end
+
+  private
+
+    def usage
+      puts "Just deploy your apps with docker and consul. Nothing else."
+      puts "Version: #{AppRb::VERSION}"
+      puts ""
+      puts "  app-rb <yml> <command>"
+      puts ""
+      puts "  deploy [hash]      - deploy new version of app"
+      puts "  status             - status of app"
+      puts "  stop               - stop app"
+    end
   end
 end

data/lib/app-rb/command.rb

@@ -0,0 +1,232 @@
+module AppRb
+  class Command
+    def initialize(config)
+      @config = config
+    end
+
+    def deploy(target)
+      @base = "#{@config.app}-#{Time.now.to_i}"
+
+      # init
+      current_hash = AppRb::Util::Consul.kv_get(@config.consul, @config.app, "hash")
+      build_nodes = @config.nodes(@config.image["constraint"])
+      pre_payloads = @config.pre_deploy.map do |pre|
+        {
+          "nodes" => @config.nodes(pre["constraint"]),
+          "cmd" => pre["cmd"],
+          "opts" => pre["opts"] || [],
+        }
+      end
+      deploy_payloads = @config.deploy.map { |key, section|
+        nodes = @config.nodes(section["constraint"])
+        {
+          "key" => key,
+          "nodes" => nodes,
+          "amount" => (section["per"] ? section["per"]*nodes.count : section["amount"]),
+          "cmd" => section["cmd"],
+          "port" => section["port"],
+          "check_url" => section["check_url"],
+          "opts" => section["opts"] || [],
+        }
+      }
+      old_ips = AppRb::Util::Consul.kv_get(@config.consul, @config.app, "nodes").split(",")
+      new_ips = (
+        build_nodes.map(&:ip) +
+        pre_payloads.flat_map { |p| p["nodes"].map(&:ip) } +
+        deploy_payloads.flat_map { |p| p["nodes"].map(&:ip) }
+      ).uniq
+      ips = (old_ips + new_ips).uniq
+      AppRb::Util::Consul.kv_set(@config.consul, @config.app, "nodes", ips.join(","))
+
+      # pre
+      new_hash = prepare_image(build_nodes, target)
+      pre_deploy(pre_payloads, new_hash)
+      stop_bg_jobs(ips)
+
+      # deploy
+      do_deploy(deploy_payloads, new_hash)
+
+      # switch
+      blue_green(deploy_payloads, new_hash)
+
+      # clean
+      stop_services(ips)
+      clean_registry(current_hash, [current_hash, new_hash].uniq)
+
+      # finish
+      AppRb::Util::Consul.kv_set(@config.consul, @config.app, "nodes", new_ips.join(","))
+      puts AppRb::Util.green("Done.")
+      if current_hash != "" && !target && current_hash != target
+        puts "to rollback fire: app-rb #{ARGV[0]} deploy #{current_hash}"
+      end
+    end
+
+    def status
+      ips = AppRb::Util::Consul.kv_get(@config.consul, @config.app, "nodes").split(",")
+      nodes = @config.nodes.select { |n| ips.index(n.ip) }
+      max_name_len = nodes.map { |n| n.name.length }.max
+      max_ip_len = nodes.map { |n| n.ip.length }.max
+      max_roles_len = nodes.map { |n| n.roles.inspect.length }.max
+      current_base = AppRb::Util::Consul.kv_get(@config.consul, @config.app, "base")
+      current_hash = AppRb::Util::Consul.kv_get(@config.consul, @config.app, "hash")
+      current_dockers = nodes.map { |n| 
+        AppRb::Util::Docker.ids(@config.user, n.ip, {app: @config.app, build: current_base}).count
+      }
+      dockers = nodes.map { |n| 
+        AppRb::Util::Docker.ids(@config.user, n.ip, {app: @config.app}).count
+      }
+      puts ""
+      puts AppRb::Util.green("App:     ") + @config.app
+      puts AppRb::Util.green("Base:    ") + current_base
+      puts AppRb::Util.green("Hash:    ") + current_hash
+      nodes.each_with_index do |n, i|
+        puts(
+          " "*5 + n.name.rjust(max_name_len) + 
+          " "*2 + n.ip.ljust(max_ip_len) + 
+          " "*2 + n.roles.inspect.ljust(max_roles_len) +
+          " "*2 + AppRb::Util.green(current_dockers[i]) + " / " + (dockers[i] - current_dockers[i] == 0 ? "0" : AppRb::Util.red(dockers[i] - current_dockers[i]))
+        )
+        end
+    end
+
+    def stop
+      ips = AppRb::Util::Consul.kv_get(@config.consul, @config.app, "nodes").split(",")
+      stop_all(ips)
+    end
+
+    def clean
+      base = AppRb::Util::Consul.kv_get(@config.consul, @config.app, "base")
+      ips = AppRb::Util::Consul.kv_get(@config.consul, @config.app, "nodes").split(",")
+      stop_services(ips, base)
+    end
+
+    def restart
+      hash = AppRb::Util::Consul.kv_get(@config.consul, @config.app, "hash")
+      raise "FATAL: app is not started?" if hash == ""
+      puts "hash=#{hash}"
+      deploy(hash)
+    end
+
+  private
+
+    def full_image_name(hash)
+      "#{@config.registry}/#{@config.app}:#{hash}"
+    end
+
+    def prepare_image(build_nodes, target)
+      puts AppRb::Util.blue("+++ CLONE or UPDATE repository")
+      AppRb::Util::Build.build(
+        @config.user, build_nodes.sample.ip,
+        @config.image["repo"], @config.image["key"], target || @config.image["target"],
+        @config.registry, @config.app, @config.image["pre_build"] || []
+      )
+    end
+
+    def pre_deploy(pre_payloads, hash)
+      pre_payloads.each_with_index do |payload, index|
+        puts AppRb::Util.blue("+++ PRE: #{payload["cmd"].inspect}")
+        AppRb::Util::Docker.run_batch(
+          @config.user, payload["nodes"].sample.ip,
+          "#{@base}-pre-#{index}", full_image_name(hash), payload["cmd"],
+          {app: @config.app, build: @base},
+          @config.env,
+          payload["opts"]
+        )
+      end
+    end
+    
+    def do_deploy(deploy_payloads, hash)
+      puts AppRb::Util.blue("+++ PULL")
+      deploy_payloads.flat_map { |p| p["nodes"] }.uniq.map { |node|
+        Thread.new do
+          AppRb::Util::Docker.pull(@config.user, node.ip, full_image_name(hash))
+        end
+      }.each(&:join)
+
+      deploy_payloads.each do |payload|
+        puts AppRb::Util.blue("+++ DEPLOY '#{payload["key"]}'")
+
+        # naive scheduling
+        plan = payload["nodes"].map{ |n| [n.ip, []] }.to_h
+        payload["amount"].times do |index|
+          ip = payload["nodes"][index % payload["nodes"].length].ip
+          plan[ip].push("#{payload["key"]}-#{index}")
+        end
+
+        payload["nodes"].map { |node|
+          Thread.new do
+            (plan[node.ip] || []).each do |name|
+              port = AppRb::Util.get_free_port(@config.user, node.ip)
+              puts "[#{node.name}] port=#{port}"
+
+              AppRb::Util::Docker.run_daemon(
+                @config.user, node.ip,
+                "#{@base}-#{name}", full_image_name(hash), payload["cmd"],
+                {app: @config.app, build: @base, has_port: (payload["port"] ? "yes" : "no")},
+                @config.env,
+                payload["opts"],
+                (payload["port"] ? {"#{node.ip}:#{port}" => payload["port"]} : {})
+              )
+
+              if payload["port"]
+                AppRb::Util::Consul.register_service(
+                  node.ip,
+                  "#{@base}-#{name}", "#{@base}-#{payload["key"]}", port, payload["check_url"] || "/",
+                  [@config.app, @base]
+                )
+              end
+            end
+          end
+        }.each(&:join)
+
+        if payload["port"]
+          puts AppRb::Util.blue("+++ CONSUL wait '#{payload["key"]}'")
+          AppRb::Util::Consul.consul_wait(@config.consul, "#{@base}-#{payload["key"]}")
+        end
+      end
+    end
+
+    def blue_green(deploy_payloads, new_hash)
+      service_payloads = deploy_payloads.select { |p| p["port"] }
+      AppRb::Util::Consul.kv_set(@config.consul, @config.app, "hash", new_hash)
+      AppRb::Util::Consul.kv_set(@config.consul, @config.app, "base", @base)
+      service_payloads.each do |payload|
+        AppRb::Util::Consul.kv_set(@config.consul, @config.app, "services/#{payload["key"]}", "#{@base}-#{payload["key"]}")
+      end
+      puts "\n" + AppRb::Util.green(">>>>>>>>>>>>>>>  BLUE/GREEN switch <<<<<<<<<<<<<<<") + "\n\n"
+      sleep 3
+      (AppRb::Util::Consul.kv_keys(@config.consul, @config.app + "/services") - service_payloads.map { |p| p["key"] }).each do |remove|
+        AppRb::Util::Consul.kv_unset(@config.consul, @config.app + "/services/#{remove}")
+      end
+    end
+
+    def stop_bg_jobs(ips)
+      puts AppRb::Util.blue("+++ STOP OLD backgroud jobs")
+      ips.each do |ip|
+        AppRb::Util::Docker.stop(@config.user, ip, {app: @config.app, has_port: "no"})
+      end
+    end
+
+    def stop_services(ips, base = @base)
+      puts AppRb::Util.blue("+++ STOP services")
+      AppRb::Util::Consul.remove_services(@config.consul, [@config.app], base)
+      ips.each do |ip|
+        AppRb::Util::Docker.stop(@config.user, ip, {app: @config.app, has_port: "yes"}, {build: base})
+      end
+    end
+
+    def stop_all(ips)
+      puts AppRb::Util.blue("+++ STOP")
+      AppRb::Util::Consul.remove_services(@config.consul, [@config.app])
+      ips.each do |ip|
+        AppRb::Util::Docker.stop(@config.user, ip, {app: @config.app})
+      end
+      AppRb::Util::Consul.kv_unset(@config.consul, @config.app)
+    end
+
+    def clean_registry(current_hash, keep_hashes = [])
+      puts AppRb::Util.blue("+++ CLEAN REGISTRY")
+      AppRb::Util::Registry.clean(@config.registry, @config.app, keep_hashes)
+    end
+  end
+end

data/lib/app-rb/config.rb

@@ -0,0 +1,38 @@
+class AppRb::Config
+  def initialize(yml)
+    @body = yml
+  end
+
+  def tool_version; @body["tool_version"]; end
+  def app; @body["app"]; end
+  def consul; @body["consul"]; end
+  def registry; @body["registry"]; end
+  def user; @body["user"]; end
+  def image; @body["image"]; end
+  def env; @body["env"] || {}; end
+  def pre_deploy; @body["pre_deploy"] || []; end
+  def deploy; @body["deploy"] || {}; end
+
+  def nodes(constraint = nil)
+    constraint ||= {}
+    out = __nodes
+    if constraint["role"]
+      out = out.select { |n| n.roles.index(constraint["role"]) }
+    end
+    if constraint["name"]
+      out = out.select { |n| n.name == constraint["name"] }
+    end
+    out
+  end
+
+private
+
+  Node = Struct.new(:name, :ip, :roles)
+  def __nodes
+    @__nodes ||= JSON.load(AppRb::Util.just_cmd("curl -s #{@body["consul"]}/v1/catalog/nodes")).sort_by { |n| 
+      n["Node"]
+    }.map { |n|
+      Node.new(n["Node"], n["Address"], (n["Meta"] || {})["roles"].to_s.split(",").reject{ |s| s.to_s.empty? })
+    }
+  end
+end

data/lib/app-rb/util/build.rb

@@ -0,0 +1,46 @@
+module AppRb::Util::Build
+  def self.build(user, host, repo, ssh_key, target, registry, image_name, pre_build_cmds = [])
+    AppRb::Util.do_it "ssh #{user}@#{host} bash <<EOF
+      set -e
+      tmpfile=\\$(mktemp /tmp/git-ssh-#{repo_cache(repo)}.XXXXXX)
+      echo '#!/bin/sh' > \\$tmpfile
+      echo 'exec /usr/bin/ssh -o StrictHostKeyChecking=no -i #{ssh_key} \"\\$@\"' >> \\$tmpfile
+      chmod +x \\$tmpfile
+      export GIT_SSH=\\$tmpfile
+      if [ -d #{repo_cache(repo)} ]; then
+        echo update cache...
+        cd #{repo_cache(repo)}
+        git checkout . && git clean -dfx && git checkout master && git pull
+        git branch | grep -v master | xargs -r git branch -D
+      else
+        echo clone...
+        git clone git@github.com:#{repo} #{repo_cache(repo)} && cd #{repo_cache(repo)}
+      fi
+      git checkout #{target}
+      rm \\$tmpfile\nEOF"
+
+    hash = AppRb::Util.just_cmd("ssh #{user}@#{host} 'cd #{repo_cache(repo)} && git rev-parse HEAD'")
+    tags = AppRb::Util::Registry.tags_list(registry, image_name)
+    puts "hash: #{hash}"
+    puts "tags: #{tags.inspect}"
+
+    unless tags.index(hash)
+      puts AppRb::Util.blue("+++ BUILD image")
+      AppRb::Util.do_it "ssh #{user}@#{host} bash <<EOF
+        set -e
+        cd #{repo_cache(repo)}
+        #{pre_build_cmds.join("\n")}
+        docker build -t #{registry}/#{image_name}:#{hash} .
+        docker push #{registry}/#{image_name}:#{hash}
+        echo Done.\nEOF"
+    end
+
+    return hash
+  end
+
+private
+
+  def self.repo_cache(repo)
+    repo.gsub("/", "-") + "-cache"
+  end
+end

data/lib/app-rb/util/consul.rb

@@ -0,0 +1,67 @@
+module AppRb::Util::Consul
+  def self.register_service(host, id, name, port, url, tags = [])
+    AppRb::Util.do_it %(curl -s -X PUT #{host}:8500/v1/agent/service/register -d'{
+      "Id": "#{id}",
+      "Name": "#{name}",
+      "Port": #{port},
+      "Tags": #{JSON.dump(tags)},
+      "Check": {
+        "DeregisterCriticalServiceAfter": "20m",
+        "Interval": "7s",
+        "HTTP": "http://#{host}:#{port}#{url}"
+      }
+    }')
+  end
+
+  def self.remove_services(consul, tags = [], except = nil)
+    JSON.load(AppRb::Util.just_cmd("curl -s #{consul}/v1/catalog/services")).select { |k, v|
+      (v & tags) == tags
+    }.keys.each do |service|
+      JSON.load(AppRb::Util.just_cmd("curl -s #{consul}/v1/catalog/service/#{service}")).each do |s|
+        if except && s["ServiceTags"].index(except)
+          # keep this service
+        else
+          AppRb::Util.do_it %(curl -s -X DELETE #{s["Address"]}:8500/v1/agent/service/deregister/#{s["ServiceID"]})
+        end
+      end
+    end
+  end
+
+  def self.consul_wait(consul, service)
+    loop do
+      url = "curl -s #{consul}/v1/health/service/#{service}"
+      statuses = JSON.load(AppRb::Util.just_cmd(url)).flat_map { |s| 
+        s["Checks"]
+      }.reject { |c| 
+        c["CheckID"] == "serfHealth"
+      }.map { |c| 
+        c["Status"]
+      }
+      puts "statuses: #{statuses.inspect}"
+      break if statuses.uniq == ["passing"] || statuses.uniq == []
+      sleep 3
+    end
+  end
+
+  CONSUL_DIR = "apps"
+
+  def self.kv_get(consul, relative_path, k)
+    AppRb::Util.just_cmd "curl -s #{consul}/v1/kv/#{CONSUL_DIR}/#{relative_path}/#{k}?raw"
+  end
+
+  def self.kv_set(consul, relative_path, k, v)
+    AppRb::Util.do_it "curl -s -X PUT #{consul}/v1/kv/#{CONSUL_DIR}/#{relative_path}/#{k} -d#{v}"
+    puts ""
+  end
+
+  def self.kv_unset(consul, relative_path)
+    AppRb::Util.do_it "curl -s -X DELETE #{consul}/v1/kv/#{CONSUL_DIR}/#{relative_path}?recurse"
+    puts ""
+  end
+
+  def self.kv_keys(consul, relative_path)
+    (JSON.load(AppRb::Util.just_cmd "curl -s #{consul}/v1/kv/#{CONSUL_DIR}/#{relative_path}?recurse") || []).map { |v|
+      v["Key"].sub("#{CONSUL_DIR}/#{relative_path}/", "")
+    }
+  end
+end

data/lib/app-rb/util/docker.rb

@@ -0,0 +1,47 @@
+require 'open3'
+
+module AppRb::Util::Docker
+  def self.pull(user, host, image)
+    Open3.popen2e("ssh #{user}@#{host} docker pull #{image}") { |i,o,w|
+      while line = o.gets do 
+        puts "[#{host}] " + line
+      end
+      raise "FATAL" unless w.value.success?
+    }
+  end
+
+  def self.run_batch(user, host, name, image, cmd, labels = {}, env = {}, opts = [])
+    AppRb::Util.do_it "ssh #{user}@#{host} docker run " +
+      labels.map { |k, v| "--label #{k}=#{v} " }.join +
+      "--name #{name} " +
+      opts.join(" ") + " " + 
+      env.map { |k, v| "-e #{k}='#{v}' " }.join +
+      "#{image} #{cmd}"
+    AppRb::Util.do_it "ssh #{user}@#{host} docker rm #{name}"
+  end
+
+  def self.run_daemon(user, host, name, image, cmd, labels = {}, env = {}, opts = [], ports = {})
+    AppRb::Util.do_it "ssh #{user}@#{host} docker run -d " +
+      "--restart unless-stopped " +
+      labels.map { |k, v| "--label #{k}=#{v} " }.join +
+      "--name #{name} " +
+      opts.join(" ") + " " + 
+      env.map { |k, v| "-e #{k}='#{v}' " }.join +
+      ports.map { |k, v| "-p #{k}:#{v} " }.join +
+      "#{image} #{cmd}"
+  end
+
+  def self.ids(user, host, labels = {})
+    filters = labels.map { |k, v| "-f label=#{k}=#{v} " }.join("")
+    AppRb::Util.just_cmd("ssh #{user}@#{host} docker ps -q #{filters}").split("\n")
+  end
+
+  def self.stop(user, host, labels = {}, except = nil)
+    keep_ids = except ? ids(user, host, labels.merge(except)) : []
+    all_ids = ids(user, host, labels)
+    if (all_ids - keep_ids).length > 0
+      AppRb::Util.do_it("ssh #{user}@#{host} docker stop #{(all_ids - keep_ids).join(" ")}")
+      AppRb::Util.do_it("ssh #{user}@#{host} docker rm #{(all_ids - keep_ids).join(" ")}")
+    end
+  end
+end

data/lib/app-rb/util/registry.rb

@@ -0,0 +1,13 @@
+module AppRb::Util::Registry
+  def self.tags_list(registry, image_name)
+    o = JSON.load(`curl -s https://#{registry}/v2/#{image_name}/tags/list`)
+    o.is_a?(Hash) && o["errors"] ? [] : o["tags"]
+  end
+
+  def self.clean(registry, image_name, keep_tags = [])
+    (tags_list(registry, image_name) - keep_tags).each do |hash|
+      digest = AppRb::Util.just_cmd("curl -s --head -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' https://#{registry}/v2/#{image_name}/manifests/#{hash} | grep Docker-Content-Digest | cut -d' ' -f2")
+      AppRb::Util.do_it "curl -s -X DELETE https://#{registry}/v2/#{image_name}/manifests/#{digest}"
+    end
+  end
+end

data/lib/app-rb/util.rb

@@ -0,0 +1,48 @@
+module AppRb::Util
+  MIN_PORT = 10_000
+  MAX_PORT = 50_000
+
+  def self.yellow(txt); "\e[0;33m#{txt}\e[0m"; end
+  def self.red(txt);    "\e[0;31m#{txt}\e[0m"; end
+  def self.green(txt);  "\e[0;32m#{txt}\e[0m"; end
+  def self.blue(txt);   "\e[0;34m#{txt}\e[0m"; end
+
+  def self.do_it(cmd)
+    puts "[exec] #{cmd}"
+    system(cmd)
+    unless $?.success?
+      puts red("FATAL :(")
+      exit 
+    end
+  end
+
+  def self.just_cmd(cmd, skip_exit_status = false)
+    puts "[exec] #{cmd}"
+    output = `#{cmd}`
+    if $?.success? || skip_exit_status
+      output.strip
+    else
+      puts red(output)
+      puts red("FATAL :(")
+      exit
+    end
+  end
+
+  def self.compare_versions(a, b)
+    parse = proc { |v| v.split(".", 3).map(&:to_i) + [v.index("-dev") ? 1 : 0] }
+    parse.call(a) <=> parse.call(b)
+  end
+
+  def self.get_free_port(user, host)
+    port = nil
+    10.times do
+      a = MIN_PORT + rand(MAX_PORT - MIN_PORT)
+      if just_cmd("ssh #{user}@#{host} ss -ln src :#{a} | fgrep -c ':#{a}'", true) == "0"
+        port = a
+        break
+      end
+    end
+    raise "Dont find free port :-((" unless port 
+    port
+  end
+end

data/lib/app-rb/version.rb

@@ -1,3 +1,3 @@
 module AppRb
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/app-rb.rb

@@ -1,5 +1,14 @@
-require "app-rb/version"
+require "pp"
+require 'json'
 require "app-rb/cli"
+require "app-rb/version"
+require "app-rb/command"
+require "app-rb/config"
+require "app-rb/util"
+require "app-rb/util/build"
+require "app-rb/util/consul"
+require "app-rb/util/docker"
+require "app-rb/util/registry"
 
 module AppRb
   def self.get_version

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: app-rb
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Alexey Vakhov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-03-24 00:00:00.000000000 Z
+date: 2017-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -73,6 +73,13 @@ files:
 - exe/app-rb
 - lib/app-rb.rb
 - lib/app-rb/cli.rb
+- lib/app-rb/command.rb
+- lib/app-rb/config.rb
+- lib/app-rb/util.rb
+- lib/app-rb/util/build.rb
+- lib/app-rb/util/consul.rb
+- lib/app-rb/util/docker.rb
+- lib/app-rb/util/registry.rb
 - lib/app-rb/version.rb
 homepage: https://github.com/uchiru/app-rb
 licenses: