dply 0.2.19 → 0.3.0

data/lib/dply/code_archive.rb

@@ -0,0 +1,123 @@
+require 'fileutils'
+require_relative 'helper'
+
+module Dply
+  class CodeArchive
+
+    include Helper
+
+    attr_reader :name, :branch
+    attr_writer :skip_depcheck
+
+    def initialize(name, revision:)
+      @name = name
+      @branch = get_branch
+      @revision = revision
+      @dir = "tmp/build_artifacts"
+      validate
+    end
+
+    def build(git: true, gnu_tar: false, &block)
+      make_dir
+      create_tar git, gnu_tar
+      instance_eval &block if block
+      add_git_commit_id
+      add_revision
+      add_archive_name
+      depcheck
+      compress
+    end
+
+    private
+
+    def get_branch
+      out = cmd "git rev-parse --abbrev-ref HEAD", return_output: true
+      out.chomp.tr("/", "_")
+    end
+
+    def tar
+      @tar ||= "#{@dir}/#{@name}-#{@revision}-#{@branch}.tar"
+    end
+
+    def make_dir
+      FileUtils.mkdir @dir if not File.directory? @dir
+    end
+
+    def create_tar(git, gnu_tar)
+      tmp_dir = "tmp/app_code"
+      if git
+        if gnu_tar
+          FileUtils.rm_rf tmp_dir if File.exists? tmp_dir
+          FileUtils.mkdir tmp_dir
+          sh "git archive HEAD | tar xf - -C #{tmp_dir}"
+          cmd "tar cf #{tar} -C #{tmp_dir} ."
+        else
+          cmd "git archive -o #{tar} HEAD"
+        end
+      else
+        cmd "tar -cf #{tar} -T /dev/null"
+      end
+    ensure
+      FileUtils.rm_rf tmp_dir if File.exists? tmp_dir
+    end
+
+    def add(file, chdir: nil)
+      if chdir
+        cmd %(tar -h -C #{chdir} --append -f "#{tar}" "#{file}")
+      else
+        cmd %(tar -h --append -f "#{tar}" "#{file}")
+      end
+    end
+
+    def remove(file)
+      cmd %(tar --delete -f "#{tar}" "#{file}")
+    end
+
+    def add_git_commit_id
+      commit_id = `git rev-parse HEAD`.chomp!
+      file = "GIT_COMMIT_ID"
+      File.write(file, commit_id)
+      add file
+    end
+
+    def add_revision
+      file = "REVISION"
+      File.write(file, @revision)
+      add file
+    end
+
+    def add_archive_name
+      file = "ARCHIVE_NAME"
+      File.write(file, @name)
+      add file
+    end
+
+    def add_bundle
+      add ".bundle"
+      add "vendor/bundle"
+    end
+
+    def depcheck
+      return if @skip_depcheck
+      command = %(ruby -W0 "#{__dir__}/scripts/depcheck.rb" #{tar})
+      logger.bullet "depcheck #{tar}"
+      cmd command, display: false
+    end
+
+    def compress
+      cmd %(#{gzip} -f -9 "#{tar}")
+    end
+
+    def gzip
+      File.exists?("/usr/bin/pigz") ? "pigz" : "gzip"
+    end
+
+    def validate
+      [:name, :branch, :revision, :dir].each do |i|
+        ivar = "@#{i}"
+        v = instance_variable_get(ivar)
+        raise Error, "archive param :#{i} is empty" if (!v || v.strip.empty?)
+      end
+    end
+  end
+end

data/lib/dply/command.rb

@@ -0,0 +1,57 @@
+require_relative 'error'
+require 'shellwords'
+
+module Dply
+  class Command
+
+    def initialize(command, env: {}, shell: false)
+      @command = command
+      @env = env
+      @shell = shell
+      validate!
+    end
+
+    def run
+      assert_success { system(@env, *command, unsetenv_others: true, 2 => 1) }
+    end
+
+    def capture
+      assert_success { IO.popen(@env, command, unsetenv_others: true) { |f| f.read } }
+    end
+
+    private
+
+    def assert_success(&block)
+      ret = yield
+      exitstatus = $?.exitstatus
+      if exitstatus != 0
+        raise Error, "non zero exit for \"#{command_str}\""
+      end
+      ret
+    end
+
+    def command
+      @shell ? command_str : command_arr
+    end
+
+    def command_arr
+      @command_arr ||= begin
+        command_arr = @command.is_a?(Array) ? @command : @command.shellsplit
+        raise Error, "empty command \"#{@command}\"" if command_arr.empty?
+        command_arr[0] = command_arr[0].shellescape if command_arr.size == 1
+        command_arr
+      end
+    end
+
+    def command_str
+      @command_str ||= @command.is_a?(String) ? @command : @command.join(" ")
+    end
+
+    def validate!
+      if @shell && @command.is_a?(Array)
+        raise Error, "command cannot be an array when shell: true"
+      end
+    end
+
+  end
+end

data/lib/dply/config_downloader.rb

@@ -8,9 +8,10 @@ module Dply
     include Helper
     attr_writer :skip_download
 
-    def initialize(config_files , base_url)
+    def initialize(config_files , base_url, dir: "config")
       @config_files = config_files
       @base_url = base_url
+      @dir = dir
       @skip_download = []
     end
 
@@ -20,7 +21,7 @@ module Dply
           logger.debug "skipping to download file #{f}"
           next
         end
-        curl.download "#{@base_url}/#{f}", "config/#{f}"
+        curl.download "#{@base_url}/#{f}", "#{@dir}/#{f}"
       end
     end
 

data/lib/dply/curl.rb

@@ -27,11 +27,7 @@ module Dply
     private
 
     def log(msg)
-      if @quiet
-        logger.debug msg
-      else
-        logger.bullet msg
-      end
+      @quiet ? logger.debug(msg) : logger.bullet(msg)
     end
 
   end

data/lib/dply/custom_logger.rb

@@ -1,5 +1,6 @@
 require 'logger'
-require 'dply/ext/string'
+require_relative 'ext/string'
+
 module Dply
   class CustomLogger < ::Logger
 
@@ -8,6 +9,7 @@ module Dply
     def initialize(file)
       super(file)
       @level = ::Logger::INFO
+#      @trace_mode = false
     end
 
     def format_message(severity, timestamp, progname, msg)
@@ -23,6 +25,21 @@ module Dply
       end
     end
 
+    def command(command, mode:)
+      case mode
+      when :arrow
+        arrow command
+      when :bullet
+        bullet command
+      else
+        debug command
+      end
+    end
+
+    def arrow(msg)
+      info "#{"\u2023".green.bold} #{msg}"
+    end
+
     def bullet(msg)
       info "#{"\u2219".bold.blue} #{msg}"
     end

data/lib/dply/deplist.rb

@@ -1,29 +1,27 @@
-require 'filemagic'
-require 'elf'
-require 'dply/helper'
-require 'dply/rpm'
-require 'dply/pkgs'
 require 'tmpdir'
+require 'set'
+require_relative 'helper'
+require_relative 'rpm'
+require_relative 'pkgs'
+require_relative 'elf'
 
 module Dply
   class Deplist
 
     include Helper
 
-    def initialize(path)
-      if Pathname.new(path).relative?
-        @path = "#{Dir.pwd}/#{path}"
-      else
-        @path = path
-      end
+    BASE_PACKAGES = Set["glibc", "libgcc", "libstdc++", "openssl", "ruby-alt", "jemalloc"]
+
+    def initialize(tar_path)
+      @tar_path = Pathname.new(tar_path).realpath
     end
 
     def verify!
-      error "#{@path} not readable" if not File.readable? @path
+      error "#{@tar_path} not readable" if not File.readable? @tar_path
       tmp_dir do
         logger.info "(in #{Dir.pwd})"
-        cmd "tar xf #{@path}"
-        pkgs_list = Pkgs.new.runtime
+        cmd "tar xf #{@tar_path}"
+        pkgs_list = Pkgs.new("pkgs.yml").runtime
 
         @libs_files_map = libs_files_map
         libs = @libs_files_map.keys
@@ -33,7 +31,7 @@ module Dply
           pp @libs_files_map
         end
 
-        deps = rpm.libs_packages_map libs
+        deps = Rpm.libs_pkgs_map libs
         verify_deps(deps, pkgs_list)
       end
     end
@@ -42,6 +40,7 @@ module Dply
 
     def verify_deps(deps, pkgs_list)
       deps.each do |lib, pkgs|
+        next if pkgs.find { |pkg| BASE_PACKAGES.include? pkg }
         if not pkgs.any? { |pkg| pkgs_list.include? pkg }
           logger.error "missing from pkgs.yml : any of #{pkgs} (lib: #{lib}, files: #{@libs_files_map[lib]})"
           @error = true
@@ -51,17 +50,6 @@ module Dply
       puts "all dependencies satisfied".green
     end
 
-    def magic
-      @magic ||= begin
-        flags = FileMagic::FLAGS_BY_SYM.select { |k,v| k.to_s =~ /no_check_/ }.keys
-        not_required_flags = [:no_check_soft, :no_check_elf, :no_check_builtin]
-        not_required_flags.each {|x| flags.delete(x) }
-        fm = FileMagic.new
-        fm.flags = flags
-        fm
-      end
-    end
-
     def tmp_dir(&block)
       dir = File.exist?("tmp") ? "tmp" : "/tmp"
       Dir.mktmpdir(nil, dir) do |d|
@@ -72,8 +60,7 @@ module Dply
     def libs_files_map
       libs = {}
       Dir["./**/*"].each do |f|
-        type = magic.file(f)
-        next if not type =~ /ELF/
+        next if not Elf.elf? f
         dynamic_libs(f).each do |l|
           libs[l] ||= []
           libs[l] << f
@@ -83,26 +70,7 @@ module Dply
     end
 
     def dynamic_libs(file)
-      path = Pathname.new(file)
-      Elf::File.open(file) do |ef|
-        return [] if not ef.has_section? ".dynamic"
-        lib_type = ef.elf_class.desc == "64-bit" ? "()(64bit)" : ""
-        dynamic = ef[".dynamic"].needed_libraries.keys
-        rpath = ef[".dynamic"].rpath.map { |i| i.sub("$ORIGIN", "#{path.dirname}") }
-        external = dynamic.reject { |l| lib_in_rpath? l, rpath }.map { |l| "#{l}#{lib_type}" }
-        logger.debug { "[#{file}]: dynamic:#{dynamic.size} rpath:#{dynamic.size - external.size} ext:#{external.size}" } if logger.debug?
-        external
-      end
-    rescue Exception
-      return []
-    end
-
-    def lib_in_rpath?(lib, rpath)
-      rpath.any? { |i| File.exist? "#{i}/#{lib}" }
-    end
-
-    def rpm
-      @rpm ||= Rpm.new
+      Elf.new(file).external_libs
     end
 
   end

data/lib/dply/deploy_config.rb

@@ -0,0 +1,34 @@
+require 'dply/base_config'
+
+module Dply
+  class DeployConfig < BaseConfig
+    define_opts do
+      opt :name
+      opt :repo
+      opt :mirror
+      opt :branch
+      opt :strategy, type: Symbol
+      opt :target
+      opt :build_url
+      opt :shared_dirs, type: Array
+      opt :config_map, type: Hash
+      opt :dir_map, type: Hash
+      opt :config_download_url
+      opt :config_skip_download, type: Array
+      opt :verify_checksum
+      opt :revision
+      opt :dir
+    end
+
+    def default_config
+      dir Dir.pwd
+      branch "master"
+      shared_dirs []
+      dir_map({
+        "tmp" => "tmp",
+        "log" => "log"
+      })
+      verify_checksum true
+    end
+  end
+end

data/lib/dply/elf.rb

@@ -0,0 +1,60 @@
+require 'elf'
+require_relative 'helper'
+
+module Dply
+  class Elf
+
+    include Helper
+
+    MAGIC_STRING = "\177ELF"
+
+    def self.elf?(path)
+      return false if not File.file? path
+      IO.binread(path, 4) == MAGIC_STRING
+    end
+
+    attr_reader :path
+
+    def initialize(path)
+      @path = Pathname.new(path)
+      @elf = ::Elf::File.new(path)
+    end
+
+    def external_libs
+      return [] if not @elf.has_section? ".dynamic"
+      dynamic = dynamic_libs()
+      external = dynamic.reject { |i| lib_in_relative_rpath? i }.map { |l| "#{l}#{lib_type}" }
+      logger.debug { "[#{@path}]: dynamic:#{dynamic.size} rpath:#{dynamic.size - external.size} ext:#{external.size}" } if logger.debug?
+      external
+    end
+
+    private
+
+    def dynamic_libs
+      @elf[".dynamic"].needed_libraries.keys
+    end
+
+    def lib_type
+      @lib_type ||= @elf.elf_class.desc == "64-bit" ? "()(64bit)" : ""
+    end
+
+    def relative_rpath
+      @relative_rpath ||= begin
+        # include both rpath and runpaths 
+        rpaths = @elf[".dynamic"].rpath + @elf[".dynamic"].runpath
+        # expand $ORIGIN
+        rpaths = rpaths.map { |i| i.sub("$ORIGIN", "#{@path.dirname}") }
+        # convert all paths to realpath
+        rpaths = rpaths.map { |i| Pathname.new(i).realpath }
+        # select paths which start with cwd or are subpaths to cwd
+        cwd = Dir.pwd
+        rpaths.select { |i| i.to_s.start_with? cwd }
+      end
+    end
+
+    def lib_in_relative_rpath?(lib)
+      relative_rpath.any? { |i| File.exist? "#{i}/#{lib}" }
+    end
+
+  end
+end