dply 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e9a3b5ef3facbc1566ddb34fa8a038d263be1b32
-  data.tar.gz: e06ca61d6ab3e7470accc884fa10f660f3652b2d
+  metadata.gz: a62731f9b97dd1e39aa3ecbc0462fdfb2896c606
+  data.tar.gz: 29f29b503deaa08c163c03d06e78ded0bf379e11
 SHA512:
-  metadata.gz: 677ce56f5aa22733fb85146fa73b5aed0ab6debce7c7f92e57dbf28287479cfc27a62973a98a9f6d6480b60ea32d5440f6bb61bbc937b16caf7c87c87a591c27
-  data.tar.gz: 3f4fff81dd674aade3902cb88b75c7a9c75b47f47189bde721e435c95e6c9fc38bfba29408f4b30a0071497d720f81b97879e9bfde999eb6786c503f6146cf48
+  metadata.gz: af99d1e9832912032f753924ba390a1c82e16999b99d25456581eb910e630b15d325006c524538839455e584e81942243ba45aa3dc0a3ef2b02ecb8e11c0e658
+  data.tar.gz: 0c3028c8e066bab4777a4695eb0e034db9c2de17c7250bb8431f762a01fb63c2f8a611b30626f41d610eec47a45afbf7c0616de84c3a721677a8f51f45af2b9c

data/Rakefile

@@ -1,2 +1,9 @@
 require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+#  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = "test/unit/*_test.rb"
+end
 

data/lib/dply/archive.rb

@@ -1,82 +1,67 @@
 require 'dply/helper'
+require 'dply/curl'
 require 'uri'
+require 'tmpdir'
+
 module Dply
   class Archive
 
     include Helper
 
+    attr_reader :name, :path, :checksum_path, :uri
+
     def initialize(url, verify_checksum: true)
-      @url = url
+      @uri = URI.parse url
       @verify_checksum = verify_checksum
+      @name = File.basename(uri.path)
+      @path = "tmp/archive/#{name}"
+      @checksum_path = "tmp/archive/#{name}.md5"
     end
 
-    def extract_to(extraction_path)
-      download_file if not @downloaded
-      FileUtils.rm_rf extraction_path if File.exists? extraction_path
-      FileUtils.mkdir_p extraction_path
-      cmd "tar xf #{path} -C #{extraction_path}", display: true
+    def extract(&block)
+      download_file
+      Dir.mktmpdir "tmp", "./" do |d|
+        extracted = "#{d}/extracted"
+        FileUtils.mkdir extracted
+        cmd "tar xf #{path} -C #{extracted}", display: true
+        yield extracted
+      end
+      cleanup
     end
 
-    def clean
-      logger.trace "cleaning cache"
-      files = [ "tmp/cache/#{name}", "tmp/cache/#{name}.md5" ]
+    private
+
+    def cleanup
+      logger.trace "cleaning tmp/archive"
+      files = [ path, checksum_path ]
       files.each { |f| FileUtils.rm f if File.exists? f }
     end
-    private
 
     def download_file
-      if File.exists? path
-        download(uri, path) if not verify_checksum
-      else
-        download(uri, path)
+      curl.download(uri, path)
+      if @verify_checksum
+        download_checksum
+        error "checksum doesn't match for archive" if not checksum_matches?
       end
-      raise if not verify_checksum
-      @downloaded = true
-    end
-    
-    def uri
-      @uri ||= URI.parse(@url)
     end
 
-    def name
-      @name ||= File.basename(uri.path)
-    end
-
-    def path
-      @path = "tmp/cache/#{name}"
+    def download_checksum
+      curl.download("#{uri}.md5", checksum_path)
     end
 
     def checksum
-      @checksum ||= load_checksum
+      File.read(checksum_path).chomp
     end
 
-    def load_checksum
-      file = "tmp/cache/#{name}.md5"
-      if File.exists? file
-        checksum = File.read(file).chomp
-        return checksum if checksum.size == 32
-      end
-      download("#{uri}.md5", file)
-      checksum = File.read(file).chomp
-      raise if checksum.size != 32
-      return checksum
-    end
-
-    def verify_checksum
-      return true if not @verify_checksum
+    def checksum_matches?
       require 'digest'
       computed_checksum = Digest::MD5.file path
       computed_checksum == checksum
     end
 
-    def download(url, outfile)
-      logger.bullet "downloading #{url} to #{outfile}"
-      http_status = `curl -w "%{http_code}" -f -s -o '#{outfile}' '#{url}' `
-      exit_status = $?.exitstatus
-      if (http_status != "200" || exit_status != 0)
-        error "failed to download #{outfile}, http status #{http_status}, exit_status #{exit_status}"
-      end
+    def curl
+      @curl ||= Curl.new
     end
-  
+
   end
 end

data/lib/dply/bundle.rb

@@ -1,20 +1,15 @@
-require 'dply/shell'
+require 'dply/helper'
 module Dply
   class Bundle
 
-    include Shell
+    include Helper
 
-    def initialize(deployment: true)
-      @deployment = deployment
+    def install_deployment
+      install(without: ["test", "development"])
     end
 
-    def install
-      return if not gemfile_exists?
-      if @deployment
-        install_deployment
-      else
-        install_global
-      end
+    def install_test
+      install(without: ["development"])
     end
 
     def rake(task, **args)
@@ -26,42 +21,38 @@ module Dply
     end
 
     def clean
+      bundle_without(without: ["development"])
       cmd "bundle clean"
     end
 
     private
 
-    def install_global
-      exitstatus = system "bundle check > /dev/null"
-      return if exitstatus
-      cmd "bundle install -j5"
+    def check
+      system "bundle check > /dev/null"
     end
 
-    def install_deployment
-      if deployment_config_present?
-        exitstatus = system "bundle check > /dev/null"
-        return if exitstatus
-      end
+    def install(without:[])
+      #persists BUNDLE_WITHOUT config
+      bundle_without(without: without)
+      return if check
       cmd "bundle install -j5 --deployment"
     end
 
-    def deployment_config_present?
-      file = ".bundle/config"
-      if not File.readable? file
-        return false
-      end
-      config = YAML.load_file file
-      config["BUNDLE_FROZEN"] == "1" && config["BUNDLE_PATH"] == "vendor/bundle"
-    rescue
-      return false
+    def bundle_without(without: [])
+      value = without.join(":")
+      cmd "bundle config --local without #{value}", return_output: true
     end
 
     def env
-      @env ||= env_from_yml
+      @env ||= begin
+        env = {}
+        env.merge! env_from_yml(".env.yml")
+        env.merge! env_from_yml("config/env.yml")
+        env
+      end
     end
 
-    def env_from_yml
-      path = "config/env.yml"
+    def env_from_yml(path)
       if not File.readable? path
         logger.debug "skipped loading env from #{path}"
         return {}

data/lib/dply/cli/depcheck.rb

@@ -15,10 +15,8 @@ module Dply
       def run
         tar_path = @argv.shift
         error "tar path not specified" if not tar_path
-        tar_path = "#{Dir.pwd}/#{tar_path}"
-        pkgs = PkgsConfig.new.pkgs
         deplist = Deplist.new(tar_path)
-        deplist.verify! pkgs
+        deplist.verify!
       end
 
     end

data/lib/dply/cli/install_pkgs.rb

@@ -1,5 +1,4 @@
-require 'dply/pkgs_config'
-require 'dply/yum'
+require 'dply/pkgs'
 require 'dply/helper'
 
 module Dply
@@ -16,8 +15,9 @@ module Dply
       def run
         opts.parse!(@argv)
         error "pkgs.yml cannot be a symlink" if File.symlink? "pkgs.yml"
-        pkgs = PkgsConfig.new(build_mode: @options[:build_mode]).pkgs
-        Yum.new(pkgs).install
+        pkgs = Pkgs.new
+        pkgs.install(build_mode: @options[:build_mode])
+        puts "installed"
       end
 
       def opts

data/lib/dply/config_downloader.rb

@@ -1,5 +1,7 @@
 require 'dply/helper'
 require 'fileutils'
+require 'dply/curl'
+
 module Dply
   class ConfigDownloader
 
@@ -19,32 +21,14 @@ module Dply
           logger.debug "skipping to download file #{f}"
           next
         end
-        download f
-        FileUtils.mv "#{tmpdir}/#{f}", "config/#{f}"
+        curl.download f, "config/#{f}"
       end
     end
 
     private
 
-    def download(file)
-      url = "#{@base_url}/#{file}"
-      logger.bullet "downloading #{file}"
-      http_status = `curl -w "%{http_code}" -f -s -o '#{tmpdir}/#{file}' '#{url}' `
-      exitstatus = $?.exitstatus
-      if (http_status != "200"  || exitstatus != 0 )
-        error "failed to download #{file}, http status #{http_status}, exitstatus #{exitstatus}"
-      end
-    end
-
-    def tmpdir
-      @tmpdir ||= "tmp/config_dl"
-    end
-
-    def init_tmpdir
-      if File.exists? tmpdir
-        FileUtils.rm_rf tmpdir
-      end
-      FileUtils.mkdir_p tmpdir
+    def curl
+      @curl ||= Curl.new
     end
 
   end

data/lib/dply/config_struct.rb

@@ -29,18 +29,18 @@ module Dply
       @build_url ||= instance_eval(&build_url_proc)
     end
 
-    def revision_proc
-      @revision_proc ||= Proc.new do
-        jenkins = Jenkins.new(repo, name)
-        jenkins.latest_successful_revision
-      end
-    end
-
     def build_url_proc
       @build_url_proc ||= Proc.new do
-        "#{repo.chomp("/")}/job/#{name}/#{revision}/artifact/build/#{name}-#{revision}-#{branch}.tar.gz"
+        "#{repo}/artifacts/#{name}/#{revision}/#{name}-#{revision}-#{branch.to_s.tr("/","_")}.tar.gz"
       end
     end
 
+    def dir_map
+      @dir_map ||= {
+        "tmp" => "tmp",
+        "log" => "log"
+      }
+    end
+
   end
 end

data/lib/dply/curl.rb

@@ -0,0 +1,38 @@
+require 'dply/helper'
+require 'fileutils'
+require 'tmpdir'
+
+module Dply
+  class Curl
+
+    include Helper
+
+    def initialize(quiet = false)
+      @quiet = quiet
+    end
+
+    def download(url, outfile)
+      Dir.mktmpdir "tmp", "./" do |d|
+        tmpfile = "#{d}/f"
+        log "downloading #{url}"
+        http_status = `curl -w "%{http_code}" -f -s -o '#{tmpfile}' '#{url}' `
+        exit_status = $?.exitstatus
+        if (http_status != "200" || exit_status != 0)
+          error "failed to download #{outfile}, http status #{http_status}, exit_status #{exit_status}"
+        end
+        FileUtils.mv tmpfile, outfile
+      end
+    end
+
+    private
+
+    def log(msg)
+      if @quiet
+        logger.debug msg
+      else
+        logger.bullet msg
+      end
+    end
+
+  end
+end

data/lib/dply/custom_logger.rb

@@ -24,22 +24,26 @@ module Dply
     end
 
     def bullet(msg)
-      puts "#{"\u2219".bold.blue} #{msg}"
+      info "#{"\u2219".bold.blue} #{msg}"
     end
 
     def trace(msg)
       return if not @trace_mode
-      puts %(#{"T".bold.blue} #{msg}\n)
+      info %(#{"T".bold.blue} #{msg}\n)
     end
 
     def remote(msg)
       return if not @remote_mode
-      puts %{dply_msg|#{msg}}
+      info %{dply_msg|#{msg}}
     end
 
     def marker(msg)
       return if not @enable_markers
-      puts "dply_marker:#{msg}"
+      info "dply_marker:#{msg}"
+    end
+
+    def silence!
+      @logdev = nil
     end
 
   end

data/lib/dply/deplist.rb

@@ -2,6 +2,7 @@ require 'filemagic'
 require 'elf'
 require 'dply/helper'
 require 'dply/rpm'
+require 'dply/pkgs'
 require 'tmpdir'
 
 module Dply
@@ -10,17 +11,34 @@ module Dply
     include Helper
 
     def initialize(path)
-      @path = path
+      if Pathname.new(path).relative?
+        @path = "#{Dir.pwd}/#{path}"
+      else
+        @path = path
+      end
     end
 
-    def deps
-      @deps ||= load_deps
+    def verify!
+      error "#{@path} not readable" if not File.readable? @path
+      tmp_dir do
+        logger.info "(in #{Dir.pwd})"
+        cmd "tar xf #{@path}"
+        pkgs_list = Pkgs.new.runtime
+
+        @libs_files_map = libs_files_map
+        libs = @libs_files_map.keys
+
+        deps = rpm.libs_packages_map libs
+        verify_deps(deps, pkgs_list)
+      end
     end
 
-    def verify!(pkgs_list)
-      deps.each do |pkgs|
+    private
+
+    def verify_deps(deps, pkgs_list)
+      deps.each do |lib, pkgs|
         if not pkgs.any? { |pkg| pkgs_list.include? pkg }
-          logger.error "missing from pkgs.yml : any of #{pkgs}"
+          logger.error "missing from pkgs.yml : any of #{pkgs} (lib: #{lib}, files: #{@libs_files_map[lib]})"
           @error = true
         end
       end
@@ -28,8 +46,6 @@ module Dply
       puts "all dependencies satisfied".green
     end
 
-    private
-
     def magic
       @magic ||= begin
         flags = FileMagic::FLAGS_BY_SYM.select { |k,v| k.to_s =~ /no_check_/ }.keys
@@ -41,17 +57,6 @@ module Dply
       end
     end
 
-    def load_deps
-      error "#{@path} not readable" if not File.readable? @path
-      tmp_dir do
-        logger.info "(in #{Dir.pwd})"
-        cmd "tar xf #{@path}"
-        @libs = get_libs
-        logger.debug @libs.inspect
-        @deps = rpm.libs_to_packages @libs
-      end
-    end
-
     def tmp_dir(&block)
       dir = File.exist?("tmp") ? "tmp" : "/tmp"
       Dir.mktmpdir(nil, dir) do |d|
@@ -59,12 +64,14 @@ module Dply
       end
     end
 
-    def get_libs
-      libs = Set.new
+    def libs_files_map
+      libs = {}
       Dir["./**/*"].each do |f|
         type = magic.file(f)
-        if type =~ /ELF/
-          dynamic_libs(f).each { |k| libs << k }
+        next if not type =~ /ELF/
+        dynamic_libs(f).each do |l|
+          libs[l] ||= []
+          libs[l] << f
         end
       end
       return libs

data/lib/dply/helper.rb

@@ -1,18 +1,58 @@
-require 'dply/shell'
-require 'dply/git'
 require 'dply/error'
+require 'dply/logger'
+require 'fileutils'
+require 'tmpdir'
 
 module Dply
   module Helper
 
-    def self.git
-      Git
+    def cmd(command, display: true, error_msg: nil, return_output: false, env:{}, shell: false)
+      stringify_values!(env)
+      if display
+        logger.bullet command
+      else
+        logger.debug command
+      end
+      command_arr = command.split
+      run_command = shell ? command : command_arr
+
+      output = if return_output
+        IO.popen(env, run_command) { |f| f.read }
+      else
+        system(env, *run_command, 2 => 1)
+      end
+      return_value = $?.exitstatus
+      error_msg ||= "non zero exit for \"#{command}\""
+      error error_msg if return_value != 0
+      return output
+    end
+
+    def symlink(src, dst)
+      if File.symlink? dst
+        Dir.mktmpdir("sym-", "./") do |d|
+          dst_tmp = "#{d}/#{File.basename dst}"
+          FileUtils.ln_s src, dst_tmp
+          cmd "mv #{dst_tmp} #{File.dirname dst}", display: false
+        end
+      elsif File.exist? dst
+        error "cannot create symlink #{dst} => #{src}"
+      else
+        FileUtils.ln_s src, dst
+      end
     end
 
-    include Shell
+    def stringify_values!(hash)
+      hash.each do |k,v|
+        hash[k] = v.to_s
+      end
+    end
+
+    def logger
+      Logger.logger
+    end
 
     def git
-      ::Dply::Helper.git
+      Git
     end
 
     def error(msg)

data/lib/dply/logger.rb

@@ -3,6 +3,10 @@ require 'dply/custom_logger'
 module Dply
   module Logger
 
+    class << self
+      attr_writer :logger
+    end
+
     def self.logger
       @logger ||= ::Dply::CustomLogger.new(STDOUT)
     end

data/lib/dply/pkgs.rb

@@ -0,0 +1,60 @@
+require 'yaml'
+require 'dply/helper'
+require_relative 'yum'
+
+module Dply
+  class Pkgs
+
+    include Helper
+
+    attr_reader :runtime, :build, :all
+
+    def initialize(pkgs_yml = nil)
+      @pkgs_yml = pkgs_yml || "pkgs.yml"
+      read_config
+    end
+
+    def install(build_mode: false, sudo: false)
+      pkgs = build_mode ? @all : @runtime
+      Yum.new(pkgs, sudo: sudo).install
+    end
+
+    def installed?(build_mode: false)
+      pkgs = build_mode ? @all : @runtime
+      Yum.new(pkgs).installed?
+    end
+
+    private
+
+    def read_config
+      @read ||= begin
+        config = load_yml
+        @runtime = config["pkgs"] || []
+        @build = config["build_pkgs"] || []
+        @all = @runtime + @build
+        @all.each { |i| validate! i }
+        true
+      end
+    end
+
+    def load_yml
+      if not File.readable? @pkgs_yml
+        logger.debug "skipping yum pkgs"
+        return {}
+      end
+      YAML.safe_load(File.read(@pkgs_yml)) || {}
+    rescue => e
+      error "error loading pkgs list" 
+    end
+
+    def validate!(pkg)
+      msg = "invalid pkg name #{pkg}"
+      error msg if pkg =~ /\.rpm\z/i
+      error msg if pkg =~/\A[A-Za-z_0-9\.\-]\z/
+      return true
+    end
+
+  end
+end
+
+

data/lib/dply/release.rb

@@ -10,25 +10,27 @@ module Dply
     attr_accessor :url, :verify_checksum
     attr_writer :name
 
-    def self.find_or_create(revision, **kwargs)
-      release = new(revision, **kwargs)
-      name = find_installed_name(revision, **kwargs)
+    def self.find_or_create(**kwargs)
+      release = new(**kwargs)
+      name = find_installed_name(**kwargs)
       release.name = name if name
       return release
     end
 
-    def self.find_installed_name(revision, **kwargs)
-      branch = kwargs.fetch(:branch).to_s.gsub(/-/, "_").sub("/", "_")
-      app_name = kwargs.fetch(:app_name).to_s.gsub(/-/, "_")
+    def self.find_installed_name(**kwargs)
+      branch = kwargs.fetch(:branch).to_s.tr('-/', '__')
+      app_name = kwargs.fetch(:app_name).to_s.tr('-/', '__')
+      revision = kwargs.fetch(:revision)
+
       name_without_ts = "#{revision}-#{app_name}-#{branch}-"
       latest = Dir["releases/#{name_without_ts}*"].sort_by { |x, y| File.mtime(x) }.first
       latest ? File.basename(latest) : nil
     end
 
-    def initialize(revision, app_name: nil, branch: nil, url: nil)
+    def initialize(revision:, app_name:, branch:, url:)
       @revision = revision
-      @branch = branch.to_s.sub("/", "_")
-      @app_name = app_name
+      @branch = branch.to_s.tr('-/', '__')
+      @app_name = app_name.to_s.tr('-/', '__')
       @url = url
     end
 
@@ -39,7 +41,7 @@ module Dply
     end
 
     def name
-      @name ||= "#{@revision}-#{replace_dashes(@app_name)}-#{replace_dashes(@branch)}-#{timestamp}"
+      @name ||= "#{@revision}-#{@app_name}-#{@branch}-#{timestamp}"
     end
 
     def install
@@ -47,12 +49,9 @@ module Dply
         logger.debug "release #{name} already installed"
         return
       end
-      Dir.mktmpdir "tmp" do |d|
-        path = "#{d}/#{name}"
-        archive.extract_to path
-        FileUtils.mv path, "releases/"
+      archive.extract do |path|
+        FileUtils.mv path, "releases/#{name}"
       end
-      archive.clean
     end
 
     def path
@@ -74,10 +73,6 @@ module Dply
 
     private
 
-    def replace_dashes(str)
-      str.to_s.gsub(/-/, "_")
-    end
-
     def archive
       @archive ||= Archive.new(url, verify_checksum: @verify_checksum)
     end

data/lib/dply/rpm.rb

@@ -4,15 +4,17 @@ module Dply
   class Rpm
     include Helper
 
-    def libs_to_packages(libs)
-      packages = Set.new
-      libs.each do |l|
-        list = whatprovides(l)
-        packages << list if not list.empty?
+    def libs_packages_map(libs)
+      h = {}
+      libs.each do |lib|
+        list = whatprovides(lib)
+        h[lib] = list if not list.empty?
       end
-      return packages
+      return h
     end
 
+    private
+
     def filtered?(pkg)
       @filtered ||= ["glibc", "libgcc", "libstdc++", "openssl", "ruby-alt", "jemalloc"]
       @filtered.include? pkg.strip
@@ -21,9 +23,7 @@ module Dply
     def whatprovides(lib)
       lib = "#{lib}()(64bit)"
       command = %(rpm --queryformat "%{NAME} " -q --whatprovides "#{lib}")
-      logger.debug command
-      output = `#{command}`
-      error "running command #{command}" if not $?.exitstatus == 0
+      output = cmd command, return_output: true, shell: true
       list = output.strip.split.select {|pkg| not filtered? pkg }
     end
 

data/lib/dply/strategy/archive.rb

@@ -84,7 +84,8 @@ module Dply
 
       def release
         @release ||= Release.find_or_create(
-          revision, branch: branch,
+          revision: revision,
+          branch: branch,
           app_name: config.name,
           url: config.build_url
         )
@@ -118,7 +119,7 @@ module Dply
       end
 
       def tasks
-        @tasks ||= Tasks.new(deployment: true)
+        @tasks ||= Tasks.new
       end
 
       def prune_releases

data/lib/dply/strategy/git.rb

@@ -89,7 +89,7 @@ module Dply
       end
 
       def tasks
-        @tasks ||= Tasks.new(deployment: true)
+        @tasks ||= Tasks.new
       end
 
     end

data/lib/dply/tasks.rb

@@ -1,38 +1,32 @@
 require 'json'
-require 'dply/shell'
+require 'dply/helper'
 require 'dply/bundle'
-require 'dply/yum'
 require 'dply/linker'
-require 'dply/pkgs_config'
-require 'dply/error'
+require 'dply/pkgs'
 require 'etc'
 
 module Dply
   class Tasks
 
-    include Shell
-
-    def initialize(deployment: true)
-      @deployment = deployment
-    end
+    include Helper
 
     def deploy(target)
-      bundle.install
+      bundle.install_deployment
       rake "#{target}:deploy"
     end
 
     def reload(target)
-      bundle.install
+      bundle.install_deployment
       rake "#{target}:reload"
     end
 
     def task(task)
-      bundle.install
+      bundle.install_deployment
       rake task
     end
 
     def build(task)
-      bundle.install
+      bundle.install_deployment
       bundle.clean
       rake task
     end
@@ -50,13 +44,12 @@ module Dply
 
     def install_pkgs(build_mode: false, use_yum: false)
       return if not File.exists? "pkgs.yml"
+      return if pkgs.installed?(build_mode: build_mode)
       drake_exists = File.exists? (drake_command)
 
-      pkgs = get_pkgs(build_mode)
       if use_yum || !drake_exists
-        yum_install pkgs
+        pkgs.install(build_mode: build_mode, sudo: true)
       else
-        return if Yum.new(pkgs).installed?
         command_install build_mode
       end
     end
@@ -72,15 +65,11 @@ module Dply
     private
 
     def bundle
-      @bundle ||= Bundle.new(deployment: @deployment)
-    end
-
-    def get_pkgs(build_mode)
-      PkgsConfig.new(build_mode: build_mode).pkgs
+      @bundle ||= Bundle.new
     end
 
-    def yum_install(pkgs)
-      Yum.new(pkgs, sudo: true).install
+    def pkgs
+      @pkgs ||= Pkgs.new
     end
 
     def command_install(build_mode)

data/lib/dply/version.rb

@@ -1,3 +1,3 @@
 module Dply
-  VERSION = "0.2.3"
+  VERSION = "0.2.4"
 end

data/lib/dply/yum.rb

@@ -6,11 +6,7 @@ module Dply
     include Helper
 
     def initialize(pkgs, sudo: false)
-      if pkgs.is_a? Set
-        @pkgs = pkgs.to_a
-      else
-        @pkgs = pkgs
-      end
+      @pkgs = pkgs
       @sudo = sudo
     end
 

data/lib/dplyr/cli.rb

@@ -23,7 +23,14 @@ module Dplyr
       when 'local'
         system "drake #{global_switches.join(" ")} #{argv_str}"
       else
-        run_remote_task
+        command = argv[0]
+        case command
+        when "list"
+          require 'pp'
+          pp hosts
+        else
+          run_remote_task
+        end
       end
     end
 

data/lib/dplyr/consul.rb

@@ -0,0 +1,28 @@
+require 'open-uri'
+require 'json'
+require 'dply/error'
+
+module Dplyr
+  class Consul
+
+    def hosts(app_name, service: "app")
+      uri = "http://127.0.0.1:8500/v1/catalog/service/#{service}?tag=#{app_name}"
+      response = JSON.parse(open(uri).read)
+      hosts = []
+      response.each do |i|
+        host = {}
+        metadata_tag = i["ServiceTags"].find {|t| t =~ /\Ametadata:/}
+        metadata = metadata_tag ? JSON.parse(metadata_tag.partition(":")[2]) : {}
+        host[:user] = metadata["user"]
+        host[:dir] = metadata["dir"]
+        host[:addr] = i["Address"] 
+        host[:id] = i["Node"]
+        hosts << host
+      end
+      hosts
+    rescue
+      raise ::Dply::Error, "failed to load hosts from consul"
+    end
+
+  end
+end

data/lib/dplyr/stage.rb

@@ -1,4 +1,5 @@
 require 'dply/helper'
+require_relative 'consul'
 
 module Dplyr
   class Stage
@@ -11,6 +12,8 @@ module Dplyr
       @name = name.to_sym
       @hosts = []
       @parallel_runs = 1
+      @dir = nil
+      @user = nil
     end
 
     def data
@@ -44,18 +47,31 @@ module Dplyr
       @parallel_runs = parallel_runs
     end
 
-    def fill_hosts
-      @hosts.each do |host|
-        host[:user] ||= fetch(:user)
-        host[:dir] ||= fetch(:dir)
+    def consul(app_name, service: "app")
+      consul = Consul.new
+      consul.hosts(app_name, service: service).each do |i|
+        host(i[:addr], user: i[:user], dir: i[:dir], id: i[:id])
       end
     end
 
     alias_method :deploy_dir, :dir
 
+    private
+
+    def fill_hosts
+      @hosts.each do |host|
+        begin
+          host[:user] ||= fetch(:user)
+          host[:dir] ||= fetch(:dir)
+        rescue
+          error "user/dir not specified for #{host[:id]}"
+        end
+      end
+    end
+
     def fetch(var)
       value = instance_variable_get("@#{var}")
-      raise "error accessing var #{var} for stage #{name}" if not value
+      error "error accessing var #{var} for stage #{name}" if not value
       return value
     end
 

data/lib/dplyr/task_runner.rb

@@ -38,9 +38,9 @@ module Dplyr
 
     def run_in_parallel
       if @auto_serialize
-        execute_serially hosts[0]
-        execute_in_parallel Range.new(1,hosts.size - 2)
-        execute_serially hosts[-1]
+        t = execute_serially hosts[0]
+        return if t.exit_status != 0
+        execute_in_parallel Range.new(1,hosts.size - 1)
       else
         execute_in_parallel Range.new(0, hosts.size - 1)
       end

data/test/sample_data/build.tar.gz.md5

@@ -0,0 +1 @@
+642071a065e730cdb86fa3b6892ff619

data/test/test_helper.rb

@@ -0,0 +1,23 @@
+require_relative 'webserver'
+require 'minitest/reporters'
+require 'minitest/autorun'
+
+class Minitest::Test
+
+  def self.test(name, &block)
+    method_name = "test_: #{name}".to_sym
+    define_method method_name, &block
+  end
+
+end
+
+
+Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new
+
+require 'fileutils'
+FileUtils.mkdir_p "tmp/archive"
+
+if not ENV['DEBUG']
+  require 'dply/logger'
+  Dply::Logger.logger.silence!
+end

data/test/unit/archive_test.rb

@@ -0,0 +1,34 @@
+require 'test_helper'
+require 'dply/archive'
+
+module Dply
+  class ArchiveTest < Minitest::Test
+    def setup
+      url = "http://127.0.0.1:8000/build.tar.gz"
+      @name = "build.tar.gz"
+      @path = "tmp/archive/build.tar.gz"
+      @checksum_path = "tmp/archive/build.tar.gz.md5"
+      @archive = Archive.new(url, verify_checksum: true)
+    end
+
+    def teardown
+      @archive = nil
+    end
+
+    test "#new" do
+      assert_equal @archive.name, @name
+      assert_equal @archive.path, @path
+      assert_equal @archive.checksum_path, @checksum_path
+    end
+
+    test ".extract" do
+      @archive.extract do |d|
+        assert File.exist? "#{d}/code"
+      end
+      [@path, @checksum_path].each do |f|
+        refute File.exist?(f), msg: "cache file not deleted #{f}"
+      end
+    end
+
+  end
+end

data/test/unit/curl_test.rb

@@ -0,0 +1,24 @@
+require 'test_helper'
+require 'dply/curl'
+require 'tmpdir'
+
+module Dply
+  class CurlTest < Minitest::Test
+
+    def url
+      @url ||= "http://127.0.0.1:8000/build.tar.gz"
+    end
+
+    test "#download" do
+
+      Dir.mktmpdir do |dir|
+        f = "#{dir}/f"
+        curl = Curl.new
+        curl.download(url, f)
+        assert File.exist? f
+      end
+    end
+
+
+  end
+end

data/test/webserver.rb

@@ -0,0 +1,20 @@
+require 'webrick'
+class ::WEBrick::BasicLog
+  def log(level, data)
+    # nop
+  end
+end
+port = 8000
+webserver = Thread.new do
+  WEBrick::HTTPServer.new(
+    :Port => port, 
+    :DocumentRoot => "#{Dir.pwd}/test/sample_data", 
+    Logger: WEBrick::Log.new("/dev/null"),
+    AccessLog: []
+  ).start
+end
+puts "staring webserver on port #{port}"
+webserver.run
+
+at_exit { Thread.kill webserver }
+ 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dply
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Neeraj
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-10 00:00:00.000000000 Z
+date: 2015-09-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-elf
@@ -108,6 +108,7 @@ files:
 - lib/dply/config.rb
 - lib/dply/config_downloader.rb
 - lib/dply/config_struct.rb
+- lib/dply/curl.rb
 - lib/dply/custom_logger.rb
 - lib/dply/deplist.rb
 - lib/dply/error.rb
@@ -118,14 +119,13 @@ files:
 - lib/dply/linker.rb
 - lib/dply/lock.rb
 - lib/dply/logger.rb
-- lib/dply/pkgs_config.rb
+- lib/dply/pkgs.rb
 - lib/dply/release.rb
 - lib/dply/release_helper.rb
 - lib/dply/repo.rb
 - lib/dply/rpm.rb
 - lib/dply/setup.rb
 - lib/dply/shared_dirs.rb
-- lib/dply/shell.rb
 - lib/dply/strategy.rb
 - lib/dply/strategy/archive.rb
 - lib/dply/strategy/git.rb
@@ -136,12 +136,19 @@ files:
 - lib/dply/version.rb
 - lib/dply/yum.rb
 - lib/dplyr/cli.rb
+- lib/dplyr/consul.rb
 - lib/dplyr/remote_task.rb
 - lib/dplyr/report.rb
 - lib/dplyr/stage.rb
 - lib/dplyr/stages_config.rb
 - lib/dplyr/task_runner.rb
 - lib/dplyr/tasks_config.rb
+- test/sample_data/build.tar.gz
+- test/sample_data/build.tar.gz.md5
+- test/test_helper.rb
+- test/unit/archive_test.rb
+- test/unit/curl_test.rb
+- test/webserver.rb
 homepage: ''
 licenses:
 - MIT
@@ -162,8 +169,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: rake based deploy tool
-test_files: []
+test_files:
+- test/sample_data/build.tar.gz
+- test/sample_data/build.tar.gz.md5
+- test/test_helper.rb
+- test/unit/archive_test.rb
+- test/unit/curl_test.rb
+- test/webserver.rb

data/lib/dply/pkgs_config.rb

@@ -1,63 +0,0 @@
-require 'set'
-require 'yaml'
-require 'dply/helper'
-
-module Dply
-  class PkgsConfig
-
-    include Helper
-
-    def initialize(pkgs_yml = nil, build_mode: false)
-      @pkgs_yml = pkgs_yml || "pkgs.yml"
-      @build_mode = build_mode
-      @pkgs = Set.new
-    end
-
-    def pkgs
-      populate_all if not @populated
-      @pkgs
-    end
-
-    private
-
-    def config
-      @config ||= load_yml
-    end
-
-    def load_yml
-      if not File.readable? @pkgs_yml
-        logger.debug "skipping yum pkgs"
-        return {}
-      end
-      YAML.safe_load(File.read(@pkgs_yml)) || {}
-    rescue => e
-      error "error loading pkgs list" 
-    end
-
-    def populate_all
-      populate :pkgs
-      populate :build_pkgs if @build_mode
-      @populated = true
-    end
-
-    def populate(pkg_set)
-      list = config[pkg_set.to_s] || []
-      list.each { |x| add x }
-    end 
-      
-    def add(pkg)
-      pkg = pkg.strip
-      validate! pkg
-      @pkgs << pkg
-    end
-
-    def validate!(pkg)
-      msg = "invalid pkg name #{pkg}"
-      error msg if pkg =~ /\.rpm\z/i
-      error msg if pkg =~/\A[A-Za-z_0-9\.\-]\z/
-    end
-
-  end
-end
-
-

data/lib/dply/shell.rb

@@ -1,57 +0,0 @@
-require 'dply/error'
-require 'dply/logger'
-
-module Dply
-  module Shell
-
-    include Logger
-
-    def cmd(command, display: true, error_msg: nil, return_output: false, env:{})
-      stringify_values(env)
-      if display
-        puts "#{"\u2219".bold.blue} #{command}"
-      else
-        logger.debug command
-      end
-      command_arr = command.split
-
-      if return_output
-        output = IO.popen(env, command_arr) { |f| f.read }
-      else
-        output = ""
-        system(env, *command_arr, 2 => 1)
-      end 
-      return_value = $?.exitstatus
-      error_msg ||= "non zero exit for \"#{command}\""
-      raise ::Dply::Error, error_msg if return_value !=0 
-      return output
-    end 
-
-    def symlink(src, dst)
-      if File.symlink? dst
-        FileUtils.rm dst
-        FileUtils.ln_s src, dst
-      elsif File.exist? dst
-        raise "cannot create symlink #{dst} => #{src}"
-      else
-        FileUtils.ln_s src, dst
-      end
-    end
-
-    def symlink_in_dir(src,destdir)
-      if not Dir.exist? destdir
-        raise "symlink destination not a dir"
-      end
-      src_path = Pathname.new(src)
-      dst = "#{destdir}/#{src_path.basename}"
-      symlink src, dst
-    end
-
-    def stringify_values(hash)
-      hash.each do |k,v|
-        hash[k] = v.to_s
-      end
-    end
-
-  end
-end