dply 0.2.19 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. checksums.yaml +5 -5
  2. data/.rspec +4 -0
  3. data/Rakefile +0 -14
  4. data/TODO +0 -1
  5. data/code_dump/old_remote_task.rb +2 -0
  6. data/{dev_bin → dev_exe}/dplyr +0 -0
  7. data/{dev_bin → dev_exe}/drake +1 -1
  8. data/dply.gemspec +2 -2
  9. data/{bin → exe}/dplyr +0 -0
  10. data/{bin → exe}/drake +12 -14
  11. data/lib/dply/TEST_TODO +50 -0
  12. data/lib/dply/app_config.rb +108 -0
  13. data/lib/dply/base_config.rb +110 -0
  14. data/lib/dply/build.rb +17 -11
  15. data/lib/dply/build_config.rb +28 -96
  16. data/lib/dply/bundle.rb +7 -30
  17. data/lib/dply/cli/build.rb +5 -12
  18. data/lib/dply/cli/ctl.rb +7 -8
  19. data/lib/dply/cli/deploy.rb +6 -10
  20. data/lib/dply/cli/devbuild.rb +6 -10
  21. data/lib/dply/cli/install_pkgs.rb +2 -3
  22. data/lib/dply/cli/run.rb +27 -0
  23. data/lib/dply/cli/status.rb +1 -2
  24. data/lib/dply/cli/task.rb +6 -12
  25. data/lib/dply/code_archive.rb +123 -0
  26. data/lib/dply/command.rb +57 -0
  27. data/lib/dply/config_downloader.rb +3 -2
  28. data/lib/dply/curl.rb +1 -5
  29. data/lib/dply/custom_logger.rb +18 -1
  30. data/lib/dply/deplist.rb +16 -48
  31. data/lib/dply/deploy_config.rb +34 -0
  32. data/lib/dply/elf.rb +60 -0
  33. data/lib/dply/env.rb +9 -0
  34. data/lib/dply/git.rb +15 -8
  35. data/lib/dply/helper.rb +21 -33
  36. data/lib/dply/linker.rb +27 -27
  37. data/lib/dply/lock.rb +2 -9
  38. data/lib/dply/logger.rb +1 -1
  39. data/lib/dply/pkgs.rb +9 -11
  40. data/lib/dply/release.rb +2 -2
  41. data/lib/dply/{archive.rb → remote_archive.rb} +1 -1
  42. data/lib/dply/repo.rb +3 -3
  43. data/lib/dply/rpm.rb +12 -20
  44. data/lib/dply/scripts/depcheck.rb +4 -0
  45. data/lib/dply/shared_dirs.rb +1 -1
  46. data/lib/dply/strategy/archive.rb +15 -22
  47. data/lib/dply/strategy/base.rb +82 -0
  48. data/lib/dply/strategy/git.rb +18 -19
  49. data/lib/dply/task_dsl.rb +101 -0
  50. data/lib/dply/util.rb +75 -0
  51. data/lib/dply/venv.rb +53 -0
  52. data/lib/dply/version.rb +1 -1
  53. data/lib/dply/yum.rb +21 -31
  54. data/lib/dplyr/consul.rb +1 -1
  55. data/spec/dply/base_config_spec.rb +178 -0
  56. data/spec/dply/bundle_spec.rb +100 -0
  57. data/spec/dply/command_spec.rb +190 -0
  58. data/spec/dply/curl_spec.rb +41 -0
  59. data/spec/dply/deplist_spec.rb +48 -0
  60. data/spec/dply/elf_spec.rb +64 -0
  61. data/spec/dply/env_spec.rb +57 -0
  62. data/spec/dply/git_spec.rb +136 -0
  63. data/spec/dply/helper_spec.rb +168 -0
  64. data/spec/dply/linker_spec.rb +81 -0
  65. data/spec/dply/lock_spec.rb +24 -0
  66. data/spec/dply/pkgs_spec.rb +105 -0
  67. data/spec/dply/repo_spec.rb +58 -0
  68. data/spec/dply/rpm_spec.rb +32 -0
  69. data/spec/dply/yum_spec.rb +29 -0
  70. data/spec/integration/archive_flow_spec.rb +87 -0
  71. data/spec/integration/git_flow_spec.rb +63 -0
  72. data/spec/repo.rb +27 -0
  73. data/spec/spec_helper.rb +44 -0
  74. data/spec/test_data/build.tar.gz +0 -0
  75. data/spec/test_data/build.tar.gz.md5 +1 -0
  76. data/spec/test_data/bundle/gems_installed/Gemfile +1 -0
  77. data/spec/test_data/bundle/gems_not_installed/Gemfile +2 -0
  78. data/spec/test_data/bundle/no_gemfile/.gitkeep +0 -0
  79. data/spec/test_data/command/test.rb +7 -0
  80. data/spec/test_data/elf/elf +0 -0
  81. data/spec/test_data/elf/libpgtypes.so.3 +0 -0
  82. data/spec/test_data/elf/not_elf +1 -0
  83. data/spec/test_data/sample_repo/.dply.lock +0 -0
  84. data/spec/test_data/sample_repo/Gemfile +2 -0
  85. data/spec/test_data/sample_repo/Rakefile +3 -0
  86. data/spec/test_data/sample_repo/app.rb +1 -0
  87. data/spec/test_data/sample_repo/dply/app.rb +33 -0
  88. data/spec/test_data/sample_repo/lib/libacl.so.1 +0 -0
  89. data/spec/test_data/sample_repo/pkgs.yml +2 -0
  90. data/spec/webserver.rb +21 -0
  91. metadata +96 -28
  92. data/lib/dply/cli/app_task.rb +0 -38
  93. data/lib/dply/config.rb +0 -120
  94. data/lib/dply/config_struct.rb +0 -52
  95. data/lib/dply/rakelib/drake.rake +0 -33
  96. data/lib/dply/tasks.rb +0 -136
@@ -0,0 +1,9 @@
1
+ module Dply
2
+ module Env
3
+ def self.build_env(extra_env, bundled_env:)
4
+ original_env = defined?(Bundler) ? Bundler.original_env : ENV
5
+ base_env = bundled_env ? ENV : original_env
6
+ base_env.to_h.merge(extra_env)
7
+ end
8
+ end
9
+ end
@@ -7,16 +7,23 @@ module Dply
7
7
  def self.pull(branch)
8
8
  cmd "git fetch"
9
9
  checkout(branch)
10
- if tracking_branch = get_tracking_branch(branch)
10
+ if tracking_branch = tracking_branch(branch)
11
11
  cmd "git merge #{tracking_branch}"
12
12
  else
13
13
  cmd "git pull origin #{branch}"
14
14
  end
15
15
  end
16
-
16
+
17
17
  def self.checkout(branch)
18
- current_branch = `git rev-parse --abbrev-ref HEAD `.chomp.to_sym
19
- cmd "git checkout #{branch}" if branch != current_branch
18
+ current_branch = `git rev-parse --abbrev-ref HEAD `.chomp
19
+ return if branch == current_branch
20
+
21
+ local_branch_exists = system "git", "show-ref", "-q", "--verify", "refs/heads/#{branch}"
22
+ if local_branch_exists
23
+ cmd "git checkout #{branch} --"
24
+ else
25
+ cmd "git checkout -b #{branch} -t origin/#{branch}"
26
+ end
20
27
  end
21
28
 
22
29
  def self.clone(repo, dir, mirror: nil)
@@ -33,17 +40,17 @@ module Dply
33
40
  cmd "git clean -dxf "
34
41
  end
35
42
 
36
- def self.get_tracking_branch(branch)
43
+ def self.tracking_branch(branch)
37
44
  command = "git for-each-ref --format='%(upstream:short)' refs/heads/#{branch} --count=1"
38
- tracking_branch = `#{command}`
45
+ tracking_branch = cmd command, return_output: true, display: false
39
46
  if tracking_branch =~ /[a-zA-Z0-9_]/
40
47
  return tracking_branch.chomp!
41
48
  else
42
49
  return nil
43
- end
50
+ end
44
51
  end
45
52
 
46
- def self.get_remote_url
53
+ def self.remote_url
47
54
  remote_url = cmd "git config --get remote.origin.url", return_output: true, display: false
48
55
  logger.debug remote_url.chomp
49
56
  remote_url.chomp
@@ -1,44 +1,38 @@
1
- require 'dply/error'
2
- require 'dply/logger'
1
+ require_relative 'error'
2
+ require_relative 'logger'
3
+ require_relative 'command'
4
+ require_relative 'env'
3
5
  require 'fileutils'
4
6
  require 'tmpdir'
5
7
 
6
8
  module Dply
7
9
  module Helper
8
10
 
9
- def cmd(command, display: true, error_msg: nil, return_output: false, env:{}, shell: false)
10
- if command.is_a? Array
11
- command_arr = command
12
- command_str = command.join(" ")
13
- else
14
- command_arr = command.split
15
- command_str = command
16
- end
17
- stringify_values!(env)
18
- if display
19
- logger.bullet command_str
20
- else
21
- logger.debug command_str
22
- end
11
+ def cmd(command, env: {}, bundled_env: false, return_output: false, display: :bullet)
12
+ logger.command command, mode: display
13
+ new_env = Env.build_env(env, bundled_env: bundled_env)
14
+ cmd = Command.new command, env: new_env, shell: false
15
+ return_output ? cmd.capture : cmd.run
16
+ end
23
17
 
24
- run_command = shell ? command_str : command_arr
25
- output = if return_output
26
- IO.popen(env, run_command) { |f| f.read }
27
- else
28
- system(env, *run_command, 2 => 1)
29
- end
30
- return_value = $?.exitstatus
31
- error_msg ||= "non zero exit for \"#{command_str}\""
32
- error error_msg if return_value != 0
33
- return output
18
+ def sh(command, env: {}, bundled_env: false, return_output: false, display: :bullet)
19
+ logger.command command, mode: display
20
+ new_env = Env.build_env(env, bundled_env: bundled_env)
21
+ cmd = Command.new command, env: new_env, shell: true
22
+ return_output ? cmd.capture : cmd.run
34
23
  end
35
24
 
36
25
  def symlink(src, dst)
37
26
  if File.symlink? dst
27
+ # to_s to handle pathnames
28
+ return true if File.readlink(dst) == src.to_s
38
29
  Dir.mktmpdir("sym-", "./") do |d|
39
30
  dst_tmp = "#{d}/#{File.basename dst}"
40
31
  FileUtils.ln_s src, dst_tmp
41
- cmd "mv #{dst_tmp} #{File.dirname dst}", display: false
32
+
33
+ # using 'mv' here as FileUtils.mv doesn't replace a symlink which points
34
+ # to a directory
35
+ cmd ["mv", dst_tmp, File.dirname(dst)], display: false
42
36
  end
43
37
  elsif File.exist? dst
44
38
  error "cannot create symlink #{dst} => #{src}"
@@ -47,12 +41,6 @@ module Dply
47
41
  end
48
42
  end
49
43
 
50
- def stringify_values!(hash)
51
- hash.each do |k,v|
52
- hash[k] = v.to_s
53
- end
54
- end
55
-
56
44
  def logger
57
45
  Logger.logger
58
46
  end
@@ -4,53 +4,53 @@ module Dply
4
4
 
5
5
  include Helper
6
6
 
7
- attr_reader :src_dir, :dest_dir, :map
8
-
9
7
  def initialize(src_dir, dest_dir, map: {})
10
8
  verify_absolute src_dir, dest_dir
11
9
  @src_dir = src_dir
12
10
  @dest_dir = dest_dir
13
11
  @map = map
12
+ validate_map!
14
13
  end
15
-
14
+
16
15
  def create_symlinks
17
- mapped_files.each do |f|
18
- target = link_target(f)
19
- source = link_source(f)
20
- relative_source = link_relative_source(source, target)
21
- logger.debug "linking #{target} -> #{source}"
22
- error "source #{source} doesn't exist" if not File.exist? source
23
- symlink(relative_source, target)
16
+ link_pairs.each do |src, dest|
17
+ relative_source = src_relative_to_dest(src, dest)
18
+ logger.debug "linking #{dest} -> #{src}" if logger.debug?
19
+ error "source #{src} doesn't exist" if not src.exist?
20
+ symlink(relative_source, dest)
24
21
  end
25
22
  end
26
23
 
27
- def link_target(relative_target)
28
- Pathname.new "#{dest_dir}/#{relative_target}"
29
- end
24
+ private
30
25
 
31
- def link_source(relative_target)
32
- relative_source = map[relative_target]
33
- Pathname.new "#{src_dir}/#{relative_source}"
26
+ def link_pairs
27
+ Enumerator.new do |y|
28
+ @map.each do |dest, src|
29
+ src_path = Pathname.new "#{@src_dir}/#{src}"
30
+ dest_path = Pathname.new "#{@dest_dir}/#{dest}"
31
+ y.yield [src_path, dest_path]
32
+ end
33
+ end
34
34
  end
35
35
 
36
- def link_relative_source(source, target)
37
- source.relative_path_from target.parent
38
- end
39
-
40
- def mapped_files
41
- map.keys.collect do |k|
42
- path = Pathname.new k
43
- raise "config map path cannot be absoulte" if path.absolute?
44
- k
45
- end
36
+ def src_relative_to_dest(src, dest)
37
+ src.relative_path_from dest.parent
46
38
  end
47
39
 
48
40
  def verify_absolute(*paths)
49
41
  paths.each do |path|
50
42
  absolute = Pathname.new(path).absolute?
51
- raise "#{path} not absolute" if not absolute
43
+ raise Error, "#{path} not absolute" if not absolute
52
44
  end
53
45
  end
54
46
 
47
+ def validate_map!
48
+ @map.each do |dest, src|
49
+ dest_path = Pathname.new dest
50
+ src_path = Pathname.new src
51
+ raise Error, "dest path #{dest_path} not relative" if not dest_path.relative?
52
+ raise Error, "src path #{src_path} not relative" if not src_path.relative?
53
+ end
54
+ end
55
55
  end
56
56
  end
@@ -4,7 +4,7 @@ module Dply
4
4
 
5
5
  include Helper
6
6
 
7
- def initialize(dir = nil)
7
+ def initialize(dir)
8
8
  @dir = dir
9
9
  end
10
10
 
@@ -15,14 +15,7 @@ module Dply
15
15
  end
16
16
 
17
17
  def lock_file
18
- @lock_file ||= File.open("#{dir}/.dply.lock", "a+")
18
+ @lock_file ||= File.open("#{@dir}/.dply.lock", "a+")
19
19
  end
20
-
21
- private
22
-
23
- def dir
24
- @dir ||= Dir.pwd
25
- end
26
-
27
20
  end
28
21
  end
@@ -1,5 +1,5 @@
1
1
  require 'logger'
2
- require 'dply/custom_logger'
2
+ require_relative 'custom_logger'
3
3
  module Dply
4
4
  module Logger
5
5
 
@@ -1,5 +1,5 @@
1
1
  require 'yaml'
2
- require 'dply/helper'
2
+ require_relative 'helper'
3
3
  require_relative 'yum'
4
4
 
5
5
  module Dply
@@ -9,19 +9,19 @@ module Dply
9
9
 
10
10
  attr_reader :runtime, :build, :all
11
11
 
12
- def initialize(pkgs_yml = nil)
13
- @pkgs_yml = pkgs_yml || "pkgs.yml"
12
+ def initialize(pkgs_yml)
13
+ @pkgs_yml = pkgs_yml
14
14
  read_config
15
15
  end
16
16
 
17
- def install(build_mode: false, sudo: false)
17
+ def install(build_mode: false)
18
18
  pkgs = build_mode ? @all : @runtime
19
- Yum.new(pkgs, sudo: sudo).install
19
+ Yum.install pkgs
20
20
  end
21
21
 
22
22
  def installed?(build_mode: false)
23
23
  pkgs = build_mode ? @all : @runtime
24
- Yum.new(pkgs).installed?
24
+ Yum.installed? pkgs
25
25
  end
26
26
 
27
27
  private
@@ -44,18 +44,16 @@ module Dply
44
44
  return {}
45
45
  end
46
46
  YAML.safe_load(File.read(@pkgs_yml)) || {}
47
- rescue => e
48
- error "error loading pkgs list"
47
+ rescue
48
+ error "error loading pkgs list"
49
49
  end
50
50
 
51
51
  def validate!(pkg)
52
52
  msg = "invalid pkg name #{pkg}"
53
53
  error msg if pkg =~ /\.rpm\z/i
54
- error msg if pkg =~/\A[A-Za-z_0-9\.\-]\z/
54
+ error msg if not pkg =~ /\A[A-Za-z_0-9\.\-]+\z/
55
55
  return true
56
56
  end
57
57
 
58
58
  end
59
59
  end
60
-
61
-
@@ -1,4 +1,4 @@
1
- require 'dply/archive'
1
+ require_relative 'remote_archive'
2
2
  require 'dply/helper'
3
3
  require 'tmpdir'
4
4
 
@@ -74,7 +74,7 @@ module Dply
74
74
  private
75
75
 
76
76
  def archive
77
- @archive ||= Archive.new(url, verify_checksum: @verify_checksum)
77
+ @archive ||= RemoteArchive.new(url, verify_checksum: @verify_checksum)
78
78
  end
79
79
 
80
80
  def timestamp
@@ -4,7 +4,7 @@ require 'uri'
4
4
  require 'tmpdir'
5
5
 
6
6
  module Dply
7
- class Archive
7
+ class RemoteArchive
8
8
 
9
9
  include Helper
10
10
 
@@ -12,7 +12,7 @@ module Dply
12
12
 
13
13
  def create
14
14
  if Dir.exist? "#{dir}/.git"
15
- raise "unable to create repo" if not verify_remote_url
15
+ raise Error, "unable to create repo (another repo already exists)" if not verify_remote_url
16
16
  else
17
17
  Git.clone upstream, dir, mirror: @mirror
18
18
  end
@@ -22,10 +22,10 @@ module Dply
22
22
 
23
23
  def verify_remote_url
24
24
  remote_url = Dir.chdir(dir) do
25
- Git.get_remote_url
25
+ Git.remote_url
26
26
  end
27
27
  remote_url == upstream
28
28
  end
29
-
29
+
30
30
  end
31
31
  end
@@ -1,32 +1,24 @@
1
- require 'dply/helper'
1
+ require_relative 'helper'
2
2
 
3
3
  module Dply
4
- class Rpm
5
- include Helper
4
+ module Rpm
5
+ extend Helper
6
6
 
7
7
  # libs should include ()(64bit) in their name if 64bit libs
8
-
9
- def libs_packages_map(libs)
8
+ def self.libs_pkgs_map(libs)
10
9
  h = {}
11
- libs.each do |lib|
12
- list = whatprovides(lib)
13
- h[lib] = list if not list.empty?
14
- end
10
+ libs.each { |lib| h[lib] = pkg_list(lib) }
15
11
  return h
16
12
  end
17
13
 
18
- private
19
-
20
- def filtered?(pkg)
21
- @filtered ||= ["glibc", "libgcc", "libstdc++", "openssl", "ruby-alt", "jemalloc"]
22
- @filtered.include? pkg.strip
23
- end
14
+ class << self
15
+ private
24
16
 
25
- def whatprovides(lib)
26
- command = ["rpm", "--queryformat", "%{NAME} ", "-q", "--whatprovides", lib]
27
- output = cmd command, return_output: true, display: false
28
- list = output.strip.split.select {|pkg| not filtered? pkg }
17
+ def pkg_list(lib)
18
+ command = ["rpm", "--queryformat", "%{NAME} ", "-q", "--whatprovides", lib]
19
+ output = cmd command, return_output: true, display: false
20
+ output.strip.split
21
+ end
29
22
  end
30
-
31
23
  end
32
24
  end
@@ -0,0 +1,4 @@
1
+ require_relative '../deplist'
2
+
3
+ deplist = Dply::Deplist.new(ARGV.shift)
4
+ deplist.verify!
@@ -9,7 +9,7 @@ module Dply
9
9
 
10
10
  def create
11
11
  FileUtils.mkdir_p dirs
12
- end
12
+ end
13
13
 
14
14
  def create_in(dir)
15
15
  Dir.chdir(dir) { create }
@@ -1,5 +1,5 @@
1
1
  require 'dply/helper'
2
- require 'dply/tasks'
2
+ require 'dply/util'
3
3
  require 'dply/setup'
4
4
  require 'dply/config_downloader'
5
5
  require 'dply/yum'
@@ -7,11 +7,10 @@ require 'dply/release'
7
7
  require 'dply/release_helper'
8
8
  require 'forwardable'
9
9
 
10
-
11
10
  module Dply
12
11
  module Strategy
13
12
  class Archive
14
-
13
+
15
14
  extend Forwardable
16
15
  include Helper
17
16
 
@@ -19,20 +18,20 @@ module Dply
19
18
  def_delegators :config, :target, :branch, :revision, :name,
20
19
  :config_map, :dir_map, :config_skip_download,
21
20
  :config_download_url, :build_url
22
-
21
+
23
22
  def initialize(config, options)
24
23
  @config = config
25
24
  @options = options || {}
26
25
  end
27
26
 
28
27
  def deploy
29
- error "revision not specified( use -r)" if not revision
28
+ error "revision not specified( use -r)" if not revision
30
29
  setup.archive
31
30
  if release.already_deployed? && release.current?
32
31
  logger.info "revision #{revision} is currently deployed"
33
32
  current_version = previous_version = get_release
34
33
  prune_releases
35
- tasks.report_changes(current_version, previous_version)
34
+ util.report_changes(current_version, previous_version)
36
35
  return
37
36
  end
38
37
  download_configs if config_download_url
@@ -40,25 +39,19 @@ module Dply
40
39
  previous_version = get_release
41
40
  release.make_current
42
41
  Dir.chdir current_dir do
43
- tasks.deploy deploy_target
42
+ util.run "deploy:#{deploy_target}"
44
43
  end
45
44
  release.record_deployment
46
45
  current_version = get_release
47
46
  prune_releases
48
- tasks.report_changes(previous_version, current_version)
47
+ util.report_changes(previous_version, current_version)
49
48
  end
50
-
49
+
51
50
  def reload
52
51
  download_configs if config_download_url
53
52
  Dir.chdir current_dir do
54
53
  link_all
55
- tasks.reload
56
- end
57
- end
58
-
59
- def task(name)
60
- Dir.chdir current_dir do
61
- tasks.rake name
54
+ util.run :reload
62
55
  end
63
56
  end
64
57
 
@@ -80,7 +73,7 @@ module Dply
80
73
  def download_configs
81
74
  files = config_map.values.uniq
82
75
  downloader = ConfigDownloader.new(files, config_download_url)
83
- downloader.skip_download = config_skip_download if config_skip_download
76
+ downloader.skip_download = config_skip_download if config_skip_download
84
77
  downloader.download_all
85
78
  end
86
79
 
@@ -98,21 +91,21 @@ module Dply
98
91
  release.install
99
92
  Dir.chdir release.path do
100
93
  link_all
101
- tasks.install_pkgs(use_yum: options[:use_yum])
94
+ util.install_pkgs
102
95
  end
103
96
  end
104
97
 
105
98
  def link_all
106
- tasks.link "#{config.dir}/shared", dir_map
107
- tasks.link "#{config.dir}/config", config_map
99
+ util.link "#{config.dir}/shared", dir_map
100
+ util.link "#{config.dir}/config", config_map
108
101
  end
109
102
 
110
103
  def setup
111
104
  @setup ||= Setup.new(@config)
112
105
  end
113
106
 
114
- def tasks
115
- @tasks ||= Tasks.new
107
+ def util
108
+ @util ||= Util.new
116
109
  end
117
110
 
118
111
  def prune_releases