hdeploy 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: eec00ee2ca9aaa249c5cc4119718de01ffad5609
+  data.tar.gz: 14adbd225cb0154c7113561afcfeebe468fdd101
+SHA512:
+  metadata.gz: f64ebdb9712e3c1e02edd45c69d6d94dad0ff6c581ebd49fc4caea3e74ab887839161666eb17dd7f8ffd4060b0e232a57deea0dcc97ce68b1ae6809ee2f71cb3
+  data.tar.gz: 56cee39bec219096382fba67d79dbe4ef41bfa9837d8e3bf0bea61af684983a2d7cadb737f9a093809ce6fc1d584d833970178c042e067f901f24130a4986ddf

data/.gitignore

@@ -0,0 +1,51 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in hdeploy.gemspec
+gemspec
+

data/README.md

@@ -0,0 +1,2 @@
+# hdeploy
+HDeploy client ruby gem

data/bin/hdeploy

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+# FIXME: this allows to load without bundle or installing the gem
+$LOAD_PATH.unshift File.expand_path(File.join(__FILE__,'../../lib'))
+
+require 'hdeploy'
+
+cli = HDeploy::CLI.new
+cli.run!
+

data/bin/hdeploy_client

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+
+# FIXME: this allows to load without bundle or installing the gem
+$LOAD_PATH.unshift File.expand_path(File.join(__FILE__,'../../lib'))
+
+require 'hdeploy'
+
+# Note: this means this can't really be done by executing
+node = HDeploy::Client.new
+
+# Fixme: slightly better error msgs etc
+
+if ARGV.length == 0
+  raise "please specify a command such symlink post_distribute_run_once post_symlink_run_once"
+else
+  if ARGV.first == 'symlink'
+    # we need special syntax: symlink app env
+    params = {}
+    STDIN.read.split(' ').each do |param|
+      k,v = param.split(':')
+      params[k] = v
+    end
+
+    client.symlink(params)
+
+  elsif %w[symlink post_distribute_run_once post_symlink_run_once].include? ARGV.first
+
+    # Decode the next ARGV stuff. This encoding k:v is the simplest I found.
+    params = {}
+    STDIN.read.split(' ').each do |param|
+      k,v = param.split(':')
+      params[k] = v
+    end
+
+    client.run_hook(ARGV[0],params)
+  else
+    client.send(ARGV.first)
+  end
+end

data/hdeploy.gemspec

@@ -0,0 +1,25 @@
+$:.push File.expand_path("../lib", __FILE__)
+require 'hdeploy/version'
+
+Gem::Specification.new do |s|
+  s.name          = "hdeploy"
+  s.version       = HDeploy::VERSION
+  s.authors       = ["Patrick Viet"]
+  s.email         = ["patrick.viet@gmail.com"]
+  s.description   = %q{HDeploy tool}
+  s.summary       = %q{no summary}
+  s.homepage      = "https://github.com/hdeploy/hdeploy"
+
+  s.files         = `git ls-files`.split($/).select{|i| not i.start_with? 'omnibus/'}
+  s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency 'json',    '~> 1.7'
+  s.add_runtime_dependency 'curb',    '~> 0.8'
+  s.add_runtime_dependency 'inifile',   '~> 2.0'  
+  s.add_runtime_dependency 'deep_clone', '~> 0.0' # For configuration
+  s.add_runtime_dependency 'deep_merge', '~> 1.1'
+
+  s.add_development_dependency 'pry', '~> 0'
+end

data/lib/hdeploy/apiclient.rb

@@ -0,0 +1,38 @@
+require 'curb'
+require 'singleton'
+
+module HDeploy
+  class APIClient
+    include Singleton
+
+    def initialize
+      @conf = Conf.instance
+
+      @c = Curl::Easy.new()
+      @c.http_auth_types = :basic
+      @c.username = @conf.api['http_user']
+      @c.password = @conf.api['http_password']
+    end
+
+    def get(url)
+      @c.url = @conf.api['endpoint'] + url
+      @c.perform
+      raise "response code for #{url} was not 200 : #{@c.response_code}" unless @c.response_code == 200
+      return @c.body_str
+    end
+
+    def put(uri,data)
+      @c.url = @config.api['endpoint'] + uri
+      @c.http_put(data)
+      raise "response code for #{url} was not 200 : #{@c.response_code} - #{@c.body_str}" unless @c.response_code == 200
+      return @c.body_str      
+    end
+
+    def delete(uri)
+      @c.url = @config.conf['global']['endpoint'] + uri
+      @c.http_delete
+      raise "response code for #{url} was not 200 : #{@c.response_code} - #{@c.body_str}" unless @c.response_code == 200
+      return @c.body_str    
+    end
+  end
+end

data/lib/hdeploy/cli.rb

@@ -0,0 +1,523 @@
+require 'hdeploy/apiclient'
+require 'json'
+require 'fileutils'
+require 'inifile'
+require 'digest'
+
+module HDeploy
+  class CLI
+
+    def initialize
+      @config = HDeploy::Conf.instance
+      @client = HDeploy::APIClient.instance
+      @domain_name = @conf['cli']['domain_name']
+      @app = @conf['cli']['default_app']
+      @env = @conf['cli']['default_env']
+      @force = false
+      @fakebuild = false
+
+
+      @conf.each do |k|
+        next unless k[0..3] == 'app:'
+        @conf[k].each do |k2,v|
+          @conf[k][k2] = File.expand_path(v) if k2 =~ /\_path$/
+        end
+      end
+    end
+
+    def run!
+      #begin
+        cmds = []
+        ARGV.each do |arg|
+          cmd = arg.split(':',2)
+          if cmd[0][0] == '_'
+            raise "you cant use cmd starting with a _"
+          end
+
+          unless respond_to?(cmd[0])
+            raise "no such command '#{cmd[0]}' in #{self.class} (#{__FILE__})"
+          end
+          cmds << cmd
+        end
+
+        cmds.each do |cmd|
+          m = method(cmd[0]).parameters
+
+          # only zero or one param
+          raise "method #{cmd[0]} takes several parameters. This is a programming mistake. Ask Patrick to edit #{__FILE__}" if m.length > 1
+
+          if m.length == 1
+            if cmd.length > 1
+              # in this case it always works
+              puts send(cmd[0],cmd[1])
+            elsif m[0][0] == :opt
+              puts send(cmd[0])
+            else
+              # This means you didn't give parameter to command that wants an option
+              raise "method #{cmd[0]} requires an option. please specify with #{cmd[0]}:parameter"
+            end
+          else
+            if cmd.length > 1
+              raise "method #{cmd[0]} does not take parameters and you gave parameter #{cmd[1]}"
+            else
+              puts send(cmd[0])
+            end
+          end
+        end
+      #rescue Exception => e
+      #  puts "ERROR: #{e}"
+      #  exit 1
+      #end
+    end
+
+    def mysystem(cmd)
+      system cmd
+      raise "error running #{cmd} #{$?}" unless $?.success?
+    end
+
+    def fab # looking for python 'fabric'
+      return @conf['cli']['fab'] if @conf['cli']['fab']
+
+      try_files = %w[
+        /usr/local/bin/fab
+        /usr/bin/fab
+        /opt/hdeploy/embedded/bin/fab
+        /opt/hdeploy/bin/fab
+      ]
+
+      try_files.each do |f|
+        return f if File.executable?(f)
+      end
+
+      raise "could not find fabric. tried #{try_files.join(' ')}"
+    end
+
+    # -------------------------------------------------------------------------
+    def app(newapp)
+      @app = newapp
+      puts "set app to #{newapp}"
+    end
+
+    def list_servers(recipe = 'common')
+      return @client.get("/srv_by_recipe/#{recipe}")
+    end
+
+    def prune_artifacts
+      c = @conf['build'][@app]
+      keepnum = c['prune'] || 5
+      keepnum = keepnum.to_i
+
+      artdir = c['artifacts']
+
+      artlist = []
+      Dir.entries(artdir).sort.each do |f|
+        if f =~ /(#{@app}\..*)\.tar\.gz$/
+          artlist << $1
+        end
+      end
+
+      distributed_by_env = JSON.parse(@client.get("/distribute/#{@app}"))
+      distributed = {}
+      distributed_by_env.each do |env,list|
+        list.each do |artname|
+          distributed[artname] = true
+        end
+      end
+
+      artlist = artlist.delete_if {|a| distributed.has_key? a }
+
+      while artlist.length > keepnum
+        art = artlist.shift
+        artfile = art + ".tar.gz"
+        puts "File.unlink #{File.join(artdir,artfile)}"
+        File.unlink File.join(artdir,artfile)
+      end
+    end
+
+    def prune_build_env
+      c = @conf[@app]
+      keepnum = c['prune_build_env'] || 2
+      keepnum = keepnum.to_i
+
+      raise "incorrect dir config" unless c['build_dir']
+      builddir = File.expand_path(c['build_dir'])
+      return unless Dir.exists?(builddir)
+      dirs = Dir.entries(builddir).delete_if{|d| d == '.' or d == '..' }.sort
+      puts "build env pruning: keeping maximum #{keepnum} builds"
+
+      while dirs.length > keepnum
+        dirtodel = dirs.shift
+        puts "FileUtils.rm_rf #{File.join(builddir,dirtodel)}"
+        FileUtils.rm_rf File.join(builddir,dirtodel)
+      end
+    end
+
+    def prune(prune_env='nowhere')
+
+      c = @conf['build'][@app]
+      prune_count = c['prune'].to_i #FIXME: integrity check.
+      raise "no proper prune count" unless prune_count >= 3 and prune_count < 20
+
+      dist = JSON.parse(@client.get("/distribute/#{@app}"))
+      if dist.has_key? prune_env
+
+        # Now we want to be careful to not eliminate any current artifact (ie. symlinked)
+        # or any target either. Usually they would both be the same obviously.
+
+        artifacts_to_keep = {}
+
+        dist_states = JSON.parse(@client.get("/distribute_state/#{@app}"))
+        dist_states.each do |dist_state|
+          if prune_env == 'nowhere'
+            # We take EVERYTHING into account
+            artifacts_to_keep[dist_state['current']] = true
+            dist_state['artifacts'].each do |art|
+              artifacts_to_keep[art] = true
+            end
+
+          elsif dist_state['env'] == prune_env
+            # Otherwise, we only take into account the current env
+            artifacts_to_keep[dist_state['current']] = true
+          end
+        end
+
+        # If the prune_env is not 'nowhere', we also want to keep the target
+        # fixme: check integrity of reply
+        artifacts_to_keep[@client.get("/target/#{@app}/#{prune_env}")] = true
+
+        if dist[prune_env].length <= prune_count
+          return "nothing to prune in env. #{prune_env}"
+        end
+
+        delete_max_count = dist[prune_env].length - prune_count
+        delete_count = 0
+
+        dist[prune_env].sort.each do |artifact|
+
+          next if artifacts_to_keep.has_key? artifact
+
+          delete_count += 1
+          if prune_env == 'nowhere'
+            # we must also delete file
+            puts @client.delete("/artifact/#{@app}/#{artifact}")
+          else
+            puts @client.delete("/distribute/#{@app}/#{prune_env}/#{artifact}")
+          end
+          break if delete_count >= delete_max_count
+        end
+
+        return ""
+      else
+        return "Nothing to prune"
+      end
+
+      prune_artifacts
+    end
+
+    def state
+      dist = JSON.parse(@client.get("/distribute/#{@app}"))
+      dist_state = JSON.parse(@client.get("/distribute_state/#{@app}"))
+      targets = JSON.parse(@client.get("/target/#{@app}"))
+
+      # What I'm trying to do here is, for each artifact from 'dist', figure where it actually is.
+      # For this, I need to know how many servers are active per env, then I can cross-reference the artifacts
+      todisplay = {}
+      dist.each do |env,artlist|
+        next if env == 'nowhere'
+        todisplay[env] = {}
+        artlist.each do |art|
+          todisplay[env][art] = []
+        end
+      end
+
+      servers_by_env = {}
+      current_links = {}
+
+      dist_state.each do |stdata|
+        env,hostname,artifacts,current = stdata.values_at('env','hostname','artifacts','current')
+
+        servers_by_env[env] = {} unless servers_by_env.has_key? env
+        servers_by_env[env][hostname] = true
+
+        current_links[env] = {} unless current_links.has_key? env
+        current_links[env][hostname] = current
+
+        artifacts.each do |art|
+          if todisplay.has_key? env
+            if todisplay[env].has_key? art
+              todisplay[env][art] << hostname
+            end
+          end
+        end
+      end
+
+      # now that we have a servers by env, we can tell for each artifact what is distributed for it, and where it's missing.
+
+      ret = "---------------------------------------------------\n" +
+            "Artifact distribution state for app #{@app}\n" +
+            "---------------------------------------------------\n\n"
+
+      ret += "Inactive: "
+      if dist['nowhere'].length == 0
+        ret += "none\n\n"
+      else
+        ret += "\n" + dist['nowhere'].collect{|art| "- #{art}"}.sort.join("\n") + "\n\n"
+      end
+
+      todisplay.each do |env,artifacts|
+        srvnum = servers_by_env[env].length
+        txt = "ENV \"#{env}\" (#{srvnum} servers)\n"
+        ret += ("-" * txt.length) + "\n" + txt + ("-" * txt.length) + "\n"
+        ret += "TARGET: " + targets[env].to_s
+
+        # Consistent targets?
+        current_by_art = {}
+        inconsistent = []
+        current_links[env].each do |srv,link|
+          inconsistent << srv if link != targets[env]
+          current_by_art[link] = [] unless current_by_art.has_key? link
+          current_by_art[link] << srv
+        end
+        if inconsistent.length > 0
+          ret += " (#{inconsistent.length}/#{servers_by_env[env].length} inconsistent servers: #{inconsistent.join(', ')})\n\n"
+        else
+          ret += " (All OK)\n\n"
+        end
+
+        # distributed artifacts. Sort by key.
+        artifacts.keys.sort.each do |art|
+          hosts = artifacts[art]
+          ret += "- #{art}"
+          ret += " (target)" if art == targets[env]
+          ret += " (current #{current_by_art[art].length}/#{servers_by_env[env].length})" if current_by_art.has_key? art
+
+          # and if it's not distributed somewhere
+          if hosts.length < servers_by_env[env].length
+            ret += " (missing on: #{(servers_by_env[env].keys - hosts).join(', ')})"
+          end
+
+          ret += "\n"
+        end
+        ret += "\n"
+      end
+
+      ret
+    end
+
+    def force
+      @force=true
+    end
+
+    def env(newenv)
+      @env = newenv
+      puts "set env to #{@env}"
+    end
+
+    def undistribute(build_tag)
+      @client.delete("/distribute/#{@app}/#{@env}/#{build_tag}")
+    end
+
+    def help
+      puts "Possible commands:"
+      puts "  env:branch"
+      puts "  build (or build:branch)"
+      puts "  app:appname"
+      puts "  distribute:nameofartifact"
+      puts "  symlink:nameofartifact"
+      puts "  list"
+      puts ""
+      puts "Example: hdeploy env:production build"
+    end
+
+    def fakebuild
+      @fakebuild = true
+    end
+
+    def initrepo
+      init()
+    end
+
+    def init
+      c = @conf['build'][@app]
+      repo = File.expand_path(c['repo'])
+
+      if !(Dir.exists?(File.join(repo,'.git')))
+        FileUtils.rm_rf repo
+        FileUtils.mkdir_p File.join(repo,'..')
+        mysystem("git clone #{c['git']} #{repo}")
+      end
+    end
+
+    def notify(msg)
+      if File.executable?('/usr/local/bin/hdeploy_hipchat')
+        mysystem("/usr/local/bin/hdeploy_hipchat #{msg}")
+      end
+    end
+
+    def build(branch = 'master')
+
+      prune_build_env
+
+      # Starting now..
+      start_time = Time.new
+
+      # Copy GIT directory
+      c = @conf['build'][@app]
+      repo = File.expand_path(c['repo'])
+
+      raise "Error in source dir #{repo}. Please run hdeploy initrepo" unless Dir.exists? (File.join(repo, '.git'))
+      directory = File.expand_path(File.join(c['build_dir'], (@app + start_time.strftime('.%Y%m%d_%H_%M_%S.'))) + ENV['USER'] + (@fakebuild? '.fakebuild' : ''))
+      FileUtils.mkdir_p directory
+
+      # Update GIT directory
+      Dir.chdir(repo)
+
+      subgit  = `find . -mindepth 2 -name .git -type d`
+      if subgit.length > 0
+        subgit.split("\n").each do |d|
+          if Dir.exists? d
+            FileUtils.rm_rf d
+          end
+        end
+      end
+
+      [
+        'git clean -xdf',
+        'git reset --hard HEAD',
+        'git clean -xdf',
+        'git checkout master',
+        'git pull',
+        'git remote show origin',
+        'git remote prune origin',
+      ].each do |cmd|
+        mysystem(cmd)
+      end
+
+      # Choose branch
+      mysystem("git checkout #{branch}")
+
+      if branch != 'master'
+        [
+          'git reset --hard HEAD',
+          'git clean -xdf',
+          'git pull'
+        ].each do |cmd|
+          mysystem(cmd)
+        end
+      end
+
+
+      # Copy GIT
+      if c['subdir'].empty?
+        mysystem "rsync -av --exclude=.git #{c['repo']}/ #{directory}/"
+      else
+        mysystem "rsync -av --exclude=.git #{c['repo']}/c['subdir']/ #{directory}/"
+      end
+
+      # Get a tag
+      gitrev = (`git log -1 --pretty=oneline`)[0..11] # not 39.
+      build_tag = @app + start_time.strftime('.%Y%m%d_%H_%M_%S.') + branch + '.' + gitrev + '.' + ENV['USER'] + (@fakebuild? '.fakebuild' : '')
+
+      notify "build start - #{ENV['USER']} - #{build_tag}"
+
+      Dir.chdir(directory)
+
+      # Write the tag in the dest directory
+      File.write 'REVISION', (gitrev + "\n")
+
+      # Run the build process # FIXME: add sanity check
+      try_files = %w[build.sh build/build.sh hdeploy/build.sh]
+      if File.exists? 'hdeploy.ini'
+        repoconf = IniFile.load('hdeploy.ini')['global']
+        try_files.unshift(repoconf['build_script']) if repoconf['build_script']
+      end
+
+      unless @fakebuild
+        build_script = false
+        try_files.each do |f|
+          if File.exists?(f) and File.executable?(f)
+            build_script = f
+            break
+          end
+        end
+
+        raise "no executable build script file. Tried files: #{try_files.join(' ')}" unless build_script
+        mysystem(build_script)
+      end
+
+      # Make tarball
+      FileUtils.mkdir_p c['artifacts']
+      mysystem("tar czf #{File.join(c['artifacts'],build_tag)}.tar.gz .")
+
+      # FIXME: upload to S3
+      register_tarball(build_tag)
+
+      notify "build success - #{ENV['USER']} - #{build_tag}"
+
+      prune_build_env
+    end
+    
+    def register_tarball(build_tag)
+      # Register tarball
+      filename = build_tag + '.tar.gz'
+      checksum = Digest::MD5.file(File.join(@conf['build'][@app]['artifacts'], filename))
+
+      @client.put("/artifact/#{@app}/#{build_tag}", JSON.pretty_generate({
+        source: "http://build.gyg.io:8502/#{filename}",
+        altsource: "",
+        checksum: checksum,
+      }))
+    end
+
+    def fulldeploy(build_tag)
+      distribute(build_tag)
+      symlink(build_tag)
+    end
+
+    def distribute(build_tag)
+      r = @client.put("/distribute/#{@app}/#{@env}",build_tag)
+      if r =~ /^OK /
+        h = JSON.parse(@client.get("/srv/by_app/#{@app}/#{@env}"))
+
+        # On all servers, do a standard check deploy.
+        system("#{_fab} -f $(hdeploy_filepath fabfile.py) -H #{h.keys.join(',')} -P host_monkeypatch:#{@domain_name} -- sudo hdeploy_node check_deploy")
+
+        # And on a single server, run the single hook.
+        hookparams = { app: @app, env: @env, artifact: build_tag, servers:h.keys.join(','), user: ENV['USER'] }.collect {|k,v| "#{k}:#{v}" }.join(" ")
+        system("#{_fab} -f $(hdeploy_filepath fabfile.py) -H #{h.keys.sample} -P host_monkeypatch:#{@domain_name} -- 'echo #{hookparams} | sudo hdeploy_node post_distribute_run_once'")
+      end
+    end
+
+    # Does this really have to exist? Or should I just put it in the symlink method?
+    def target(artid = 'someid')
+
+      # We just check if the artifact is set to be distributed to the server
+      # for the actual presence we will only check in the symlink part.
+
+      todist = JSON.parse(@client.get("/distribute/#{@app}/#{@env}"))
+      raise "artifact #{artid} is not set to be distributed for #{@app}/#{@env}" unless todist.has_key? artid
+      return @client.put("/target/#{@app}/#{@env}", artid)
+    end
+
+    def symlink(target)
+      target(target)
+
+      h = JSON.parse(@client.get("/srv/by_app/#{@app}/#{@env}"))
+
+      raise "no server with #{@app}/#{@env}" unless h.keys.length > 0
+      h.each do |host,conf|
+        if !(conf['artifacts'].include? target)
+          raise "artifact #{target} is not present on server #{host}. Please run hdeploy env:#{@env} distribute:#{target}"
+        end
+      end
+
+      # On all servers, do a standard symlink
+      system("#{_fab} -f $(hdeploy_filepath fabfile.py) -H #{h.keys.join(',')} -P host_monkeypatch:#{@domain_name} -- 'echo app:#{@app} env:#{@env} | sudo hdeploy_node symlink'")
+
+      # And on a single server, run the single hook.
+      hookparams = { app: @app, env: @env, artifact: target, servers:h.keys.join(','), user: ENV['USER'] }.collect {|k,v| "#{k}:#{v}" }.join(" ")
+      system("#{_fab} -f $(hdeploy_filepath fabfile.py) -H #{h.keys.sample} -P host_monkeypatch:#{@domain_name} -- 'echo #{hookparams} | sudo hdeploy_node post_symlink_run_once'")
+    end
+  end
+end
+

data/lib/hdeploy/client.rb

@@ -0,0 +1,326 @@
+require 'curb'
+require 'json'
+require 'fileutils'
+require 'pathname'
+require 'inifile'
+require 'pry'
+
+module HDeploy
+  class Client
+
+    def initialize
+      @conf = HDeploy::Conf.instance('./hdeploy.conf.json')
+      @conf.add_defaults({
+        'client' => {
+          'keepalive_delay' => 60,
+          'check_deploy_delay' => 60,
+          'max_run_duration' => 3600,
+          'hostname' => `/bin/hostname`.chomp,
+        }
+      })
+
+      # Check for needed configuration parameters
+      # API
+      api_params = %w[http_user http_password endpoint]
+      raise "#{@conf.file}: you need 'api' section for hdeploy client (#{api_params.join(', ')})" unless @conf['api']
+      api_params.each do |p|
+        raise "#{@conf.file}: you need param for hdeploy client: api/#{p}" unless @conf['api'][p]
+      end
+
+      # Deploy
+      raise "#{@conf.file}: you need 'deploy' section for hdeploy client" unless @conf['deploy']
+      @conf['deploy'].keys.each do |k|
+        raise "#{@conf.file}: deploy key must be in the format app:env - found #{k}" unless k =~ /^[a-z0-9\-\_]+:[a-z0-9\-\_]+$/
+      end
+
+      default_user = Process.uid == 0 ? 'www-data' : Process.uid
+      default_group = Process.gid == 0 ? 'www-data' : Process.gid
+
+      @conf['deploy'].each do |k,c|
+        raise "#{@conf.file}: deploy section '#{k}': missing symlink param" unless c['symlink']
+        c['symlink'] = File.expand_path(c['symlink'])
+
+        # FIXME: throw exception if user/group are root and/or don't exist
+        {
+          'relpath' => File.expand_path('../releases', c['symlink']),
+          'tgzpath' => File.expand_path('../tarballs', c['symlink']),
+          'user' => default_user,
+          'group' => default_group,
+        }.each do |k2,v|
+          c[k2] ||= v
+        end
+
+        # It's not a mistake to check for uid in the gid section: only root can change gid.
+        raise "You must run client as uid root if you want a different user for deploy #{k}" if Process.uid != 0 and c['user'] != Process.uid
+        raise "You must run client as gid root if you want a different group for deploy #{k}" if Process.uid != 0 and c['group'] != Process.gid
+      end
+    end
+
+    # -------------------------------------------------------------------------
+    def keepalive
+      hostname = @conf['client']['hostname']
+      c = Curl::Easy.new(@conf['api']['endpoint'] + '/srv/keepalive/' + hostname)
+      c.http_auth_types = :basic
+      c.username = @conf['api']['http_user']
+      c.password = @conf['api']['http_password']
+      c.put((@conf['client']['keepalive_delay'].to_i * 2).to_s)
+    end
+
+    def put_state
+      hostname = @conf['client']['hostname']
+
+      c = Curl::Easy.new(@conf['api']['endpoint'] + '/distribute_state/' + hostname)
+      c.http_auth_types = :basic
+      c.username = @conf['api']['http_user']
+      c.password = @conf['api']['http_password']
+
+      r = []
+
+      # Will look at directories and figure out current state
+      @conf['deploy'].each do |section,conf|
+        app,env = section.split(':')
+
+        relpath,tgzpath,symlink = conf.values_at('relpath','tgzpath','symlink')
+
+        # could be done with ternary operator but I find it more readable like that.
+        current = "unknown"
+        if File.symlink? symlink and Dir.exists? symlink
+          current = File.basename(File.readlink(symlink))
+        end
+
+        # For artifacts, what we want is a directory, that contains the file "READY"
+        artifacts = Dir.glob(File.join(relpath, '*', 'READY')).map{|x| File.basename(File.expand_path(File.join(x,'..'))) }
+
+        r << {
+          app: app,
+          env: env,
+          current: current,
+          artifacts: artifacts.sort,
+        }
+
+      end
+
+      puts JSON.pretty_generate(r) if ENV.has_key?'DEBUG'
+      c.put(JSON.generate(r))
+    end
+
+    def find_executable(name) #FIXME should be in some other path
+      %w[
+        /opt/hdeploy/embedded/bin
+        /opt/hdeploy/bin
+        /usr/local/bin
+        /usr/bin
+      ].each do |p|
+        e = File.join p,name
+        next unless File.exists? e
+        st = File.stat(e)
+        next unless st.uid == 0
+        next unless st.gid == 0
+        if sprintf("%o", st.mode) == '100755'
+          return e
+        else
+          warn "file #{file} does not have permissions 100755"
+        end
+      end
+      return nil
+    end
+
+    def check_deploy
+      put_state
+
+      c = Curl::Easy.new()
+      c.http_auth_types = :basic
+      c.username = @conf['api']['http_user']
+      c.password = @conf['api']['http_password']
+
+      # Now this is the big stuff
+      @conf['deploy'].each do |section,conf|
+        app,env = section.split(':') #it's already checked for syntax higher in the code
+
+        # Here we get the info.
+        # FIXME: double check that config is ok
+        relpath,tgzpath,symlink,user,group = conf.values_at('relpath','tgzpath','symlink','user','group')
+
+        # Now the release info from the server
+        c.url = @conf['api']['endpoint'] + '/distribute/' + app + '/' + env
+        c.perform
+
+        # prepare directories
+        FileUtils.mkdir_p(relpath)
+        FileUtils.mkdir_p(tgzpath)
+
+        artifacts = JSON.parse(c.body_str)
+        puts "found #{artifacts.keys.length} artifacts for #{app} / #{env}"
+
+        dir_to_keep = []
+        tgz_to_keep = []
+
+        artifacts.each do |artifact,artdata|
+          puts "checking artifact #{artifact}"
+          destdir   = File.join relpath,artifact
+          tgzfile   = File.join tgzpath,(artifact+'.tar.gz')
+          readyfile = File.join destdir,'READY'
+
+          if !(File.exists?readyfile)
+            # we have to release. let's cleanup.
+            FileUtils.rm_rf(destdir) if File.exists?(destdir)
+            count = 0
+            while count < 5 and !(File.exists?tgzfile and Digest::MD5.file(tgzfile) == artdata['checksum'])
+              count += 1
+              File.unlink tgzfile if File.exists?tgzfile
+              # FIXME: add altsource and BREAK
+              # FIXME: don't run download as root!!
+              #####
+              if f = find_executable('aria2')
+                system("#{f} -x 5 -d #{tgzpath} -o #{artifact}.tar.gz #{artdata['source']}")
+
+              elsif f = find_executable('wget')
+                system("#{f} -o #{tgzfile} #{artdata['source']}")
+
+              elsif f = find_executable('curl')
+                system("#{f} -o #{tgzfile} #{artdata['source']}")                
+
+              else
+                raise "no aria2c, wget or curl available. please install one of them."
+              end
+            end
+
+            raise "unable to download artifact" unless File.exists?tgzfile
+            raise "incorrect checksum for #{tgzfile}" unless Digest::MD5.file(tgzfile) == artdata['checksum']
+
+
+            FileUtils.mkdir_p destdir
+            FileUtils.chown user, group, destdir
+            Dir.chdir destdir
+
+            chpst = ''
+            if Process.uid == 0
+              chpst = find_executable('chpst') or raise "unable to find chpst binary"
+              chpst += " -u #{user}:#{group} "
+            end
+
+            tar = find_executable('tar')
+            system("#{chpst}#{tar} xzf #{tgzfile}") or raise "unable to extract #{tgzfile} as #{user}:#{group}"
+            File.chmod 0755, destdir
+
+            # Post distribute hook
+            run_hook('post_distribute', {'app' => app, 'env' => env, 'artifact' => artifact})
+            FileUtils.touch(File.join(destdir,'READY')) #FIXME: root?
+          end
+
+          # we only get here if previous step worked.
+          tgz_to_keep << File.expand_path(tgzfile)
+          dir_to_keep << File.expand_path(destdir)            
+        end
+
+        # check for symlink
+        symlink({'app' => app,'env' => env, 'force' => false})
+
+        # cleanup
+        if Dir.exists? conf['symlink']
+          dir_to_keep << File.expand_path(File.join(File.join(conf['symlink'],'..'),File.readlink(conf['symlink'])))
+        end
+
+        (Dir.glob(File.join conf['relpath'], '*') - dir_to_keep).each do |d|
+          puts "cleanup dir #{d}"
+          FileUtils.rm_rf d
+        end
+
+        (Dir.glob(File.join conf['tgzpath'],'*') - tgz_to_keep).each do |f|
+          puts "cleanup file #{f}"
+          File.unlink f
+        end
+
+      end
+      put_state
+    end
+
+    def run_hook(hook,params)
+      # This is a generic function to run the hooks defined in hdeploy.ini.
+      # Standard hooks are
+
+      app,env,artifact = params.values_at('app','env','artifact')
+
+      oldpwd = Dir.pwd
+
+      raise "no such app/env #{app} / #{env}" unless @conf['deploy'].has_key? "#{app}:#{env}"
+
+      relpath,user,group = @conf['deploy']["#{app}:#{env}"].values_at('relpath','user','group')
+      destdir = File.join relpath,artifact
+
+      # It's OK if the file doesn't exist
+      hdeployini = File.join destdir, 'hdeploy.ini'
+      return unless File.exists? hdeployini
+
+      # It's also OK if that hook doesn't exist
+      hdc = IniFile.load(hdeployini)['hooks']
+      return unless hdc.has_key? hook
+
+      hfile = hdc[hook]
+
+      # But if it is defined, we're gonna scream if it's defined incorrectly.
+      raise "no such file #{hfile} for hook #{hook}" unless File.exists? (File.join destdir,hfile)
+      raise "non-executable file #{hfile} for hook #{hook}" unless File.executable? (File.join destdir,hfile)
+
+      # OK let's run the hook
+      Dir.chdir destdir
+
+      chpst = ''
+      if Process.uid == 0
+        chpst = find_executable('chpst') or raise "unable to find chpst binary"
+        chpst += " -u #{user}:#{group} "
+      end
+
+      system("#{chpst}#{hfile} '#{JSON.generate(params)}'")
+      if $?.success?
+        puts "Successfully run #{hook} hook / #{hfile}"
+        Dir.chdir oldpwd
+      else
+        Dir.chdir oldpwd
+        raise "Error while running file #{hfile} hook #{hook} : #{$?} - (DEBUG: (pwd: #{destdir}): #{chpst}#{hfile} '#{JSON.generate(params)}'"
+      end
+    end
+
+    def symlink(params)
+
+      app,env = params.values_at('app','env')
+      force = true
+      if params.has_key? 'force'
+        force = params['force']
+      end
+
+      raise "no such app/env #{app} / #{env}" unless @conf['deploy'].has_key? "#{app}:#{env}"
+
+      conf = @conf['deploy']["#{app}:#{env}"]
+      link,relpath = conf.values_at('symlink','relpath')
+
+      if force or !(File.exists?link)
+        FileUtils.rm_rf(link) unless File.symlink?link
+
+        c = Curl::Easy.new(@conf['api']['endpoint'] + '/target/' + app + '/' + env)
+        c.http_auth_types = :basic
+        c.username = @conf['api']['http_user']
+        c.password = @conf['api']['http_password']
+        c.perform
+
+        target = c.body_str
+        target_relative_path = Pathname.new(File.join relpath,target).relative_path_from(Pathname.new(File.join(link,'..')))
+
+        if File.symlink?(link) and (File.readlink(link) == target_relative_path)
+          puts "symlink for app #{app} is already OK (#{target_relative_path})"
+        else
+          # atomic symlink override
+          puts "setting symlink for app #{app} to #{target_relative_path}"
+          File.symlink(target_relative_path,link + '.tmp') #FIXME: should this belong to root?
+          File.rename(link + '.tmp', link)
+          put_state
+        end
+
+        run_hook('post_symlink', {'app' => app, 'env' => env, 'artifact' => target})
+      else
+        puts "not changing symlink for app #{app}"
+      end
+    end
+
+  end
+end

data/lib/hdeploy/conf.rb

@@ -0,0 +1,56 @@
+require 'json'
+require 'deep_merge'
+require 'deep_clone'
+
+module HDeploy
+  class Conf
+
+    @@instance = nil
+    @@default_values = []
+
+    attr_reader :file
+
+    def initialize(file)
+      @file = file
+      reload
+    end
+
+    # FIXME: find a good way to set default path
+    def self.instance(path = '/opt/hdeploy/etc/hdeploy.conf.json')
+      @@instance ||= new(path)
+    end
+
+    # -------------------------------------------------------------------------
+
+    def reload
+      raise "unable to find conf file #{@file}" unless File.exists? @file
+      
+      st = File.stat(@file)
+      raise "config file #{@file} must not be a symlink" if File.symlink?(@file)
+      raise "config file #{@file} must be a regular file" unless st.file?
+      raise "config file #{@file} must have uid 0" unless st.uid == 0 or Process.uid != 0
+      raise "config file #{@file} must not allow group/others to write" unless sprintf("%o", st.mode) =~ /^100[46][04][04]/
+      
+      # Seems we have checked everything. Woohoo!
+      @conf = JSON.parse(File.read(@file))
+    end
+
+    # -------------------------------------------------------------------------
+    def [](k)
+      @conf[k] 
+    end
+    
+    # -------------------------------------------------------------------------
+    def add_defaults(h)
+      # This is pretty crappy code in that it loads stuff twice etc. But that way no re-implementing a variation of deep_merge for default stuff...
+      @@default_values << h.__deep_clone__
+
+      rebuild_conf = {}
+      @@default_values.each do |defval|
+        rebuild_conf.deep_merge!(defval)
+      end
+
+      @conf = rebuild_conf.deep_merge!(@conf)
+    end
+  end
+end

data/lib/hdeploy/version.rb

@@ -0,0 +1,4 @@
+module HDeploy
+  VERSION = "0.1.0"
+end
+

data/lib/hdeploy.rb

@@ -0,0 +1,11 @@
+require 'hdeploy/conf'
+require 'hdeploy/cli'
+require 'hdeploy/client'
+require 'hdeploy/apiclient'
+
+module HDeploy
+  def HDeploy.where_is(f)
+    File.expand_path "../#{f}", __FILE__
+  end
+end
+

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification
+name: hdeploy
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Patrick Viet
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-09-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: curb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: inifile
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: deep_clone
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+- !ruby/object:Gem::Dependency
+  name: deep_merge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: HDeploy tool
+email:
+- patrick.viet@gmail.com
+executables:
+- hdeploy
+- hdeploy_client
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- bin/hdeploy
+- bin/hdeploy_client
+- hdeploy.gemspec
+- lib/hdeploy.rb
+- lib/hdeploy/apiclient.rb
+- lib/hdeploy/cli.rb
+- lib/hdeploy/client.rb
+- lib/hdeploy/conf.rb
+- lib/hdeploy/version.rb
+homepage: https://github.com/hdeploy/hdeploy
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: no summary
+test_files: []