dapp 0.5.13 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. checksums.yaml +4 -4
  2. data/bin/dapp +3 -0
  3. data/config/en/common.yml +35 -6
  4. data/config/en/net_status.yml +20 -6
  5. data/lib/dapp.rb +53 -27
  6. data/lib/dapp/application.rb +48 -27
  7. data/lib/dapp/application/git_artifact.rb +1 -1
  8. data/lib/dapp/application/path.rb +2 -18
  9. data/lib/dapp/application/tags.rb +5 -5
  10. data/lib/dapp/build/stage/artifact.rb +8 -3
  11. data/lib/dapp/build/stage/base.rb +43 -30
  12. data/lib/dapp/build/stage/{infra_install.rb → before_install.rb} +6 -6
  13. data/lib/dapp/build/stage/before_setup.rb +27 -0
  14. data/lib/dapp/build/stage/docker_instructions.rb +12 -1
  15. data/lib/dapp/build/stage/from.rb +4 -3
  16. data/lib/dapp/build/stage/{source_1_archive.rb → ga_archive.rb} +7 -7
  17. data/lib/dapp/build/stage/{source_1_archive_dependencies.rb → ga_archive_dependencies.rb} +4 -4
  18. data/lib/dapp/build/stage/{source_base.rb → ga_base.rb} +31 -7
  19. data/lib/dapp/build/stage/{source_dependencies_base.rb → ga_dependencies_base.rb} +5 -6
  20. data/lib/dapp/build/stage/{source_5.rb → ga_latest_patch.rb} +6 -6
  21. data/lib/dapp/build/stage/install/ga_post_install_patch.rb +21 -0
  22. data/lib/dapp/build/stage/install/ga_post_install_patch_dependencies.rb +21 -0
  23. data/lib/dapp/build/stage/install/ga_pre_install_patch.rb +21 -0
  24. data/lib/dapp/build/stage/install/ga_pre_install_patch_dependencies.rb +31 -0
  25. data/lib/dapp/build/stage/install/install.rb +31 -0
  26. data/lib/dapp/build/stage/mod/artifact.rb +9 -19
  27. data/lib/dapp/build/stage/mod/group.rb +42 -0
  28. data/lib/dapp/build/stage/mod/logging.rb +62 -26
  29. data/lib/dapp/build/stage/setup/chef_cookbooks.rb +37 -0
  30. data/lib/dapp/build/stage/setup/ga_post_setup_patch.rb +25 -0
  31. data/lib/dapp/build/stage/setup/ga_post_setup_patch_dependencies.rb +31 -0
  32. data/lib/dapp/build/stage/setup/ga_pre_setup_patch.rb +25 -0
  33. data/lib/dapp/build/stage/setup/ga_pre_setup_patch_dependencies.rb +27 -0
  34. data/lib/dapp/build/stage/setup/setup.rb +31 -0
  35. data/lib/dapp/builder/base.rb +9 -6
  36. data/lib/dapp/builder/chef.rb +128 -107
  37. data/lib/dapp/builder/chef/berksfile.rb +3 -0
  38. data/lib/dapp/builder/shell.rb +4 -3
  39. data/lib/dapp/cli.rb +8 -6
  40. data/lib/dapp/cli/base.rb +6 -1
  41. data/lib/dapp/cli/bp.rb +41 -0
  42. data/lib/dapp/cli/build.rb +17 -4
  43. data/lib/dapp/cli/cleanup.rb +24 -0
  44. data/lib/dapp/cli/list.rb +2 -2
  45. data/lib/dapp/cli/push.rb +8 -9
  46. data/lib/dapp/cli/run.rb +3 -4
  47. data/lib/dapp/cli/spush.rb +20 -0
  48. data/lib/dapp/cli/stages.rb +2 -0
  49. data/lib/dapp/cli/stages/cleanup.rb +7 -3
  50. data/lib/dapp/cli/stages/flush.rb +6 -3
  51. data/lib/dapp/config/application.rb +17 -11
  52. data/lib/dapp/config/chef.rb +4 -0
  53. data/lib/dapp/config/docker.rb +3 -2
  54. data/lib/dapp/config/git_artifact.rb +1 -2
  55. data/lib/dapp/config/main.rb +5 -2
  56. data/lib/dapp/config/shell.rb +20 -16
  57. data/lib/dapp/docker_registry.rb +32 -0
  58. data/lib/dapp/docker_registry/base.rb +47 -0
  59. data/lib/dapp/docker_registry/default.rb +18 -0
  60. data/lib/dapp/docker_registry/mod/authorization.rb +62 -0
  61. data/lib/dapp/docker_registry/mod/request.rb +44 -0
  62. data/lib/dapp/error/image.rb +6 -0
  63. data/lib/dapp/error/project.rb +6 -0
  64. data/lib/dapp/error/registry.rb +6 -0
  65. data/lib/dapp/exception/registry.rb +6 -0
  66. data/lib/dapp/git_artifact.rb +6 -7
  67. data/lib/dapp/git_repo/base.rb +1 -1
  68. data/lib/dapp/git_repo/remote.rb +6 -38
  69. data/lib/dapp/helper/sha256.rb +3 -3
  70. data/lib/dapp/helper/shellout.rb +25 -7
  71. data/lib/dapp/helper/streaming.rb +1 -3
  72. data/lib/dapp/image/argument.rb +31 -18
  73. data/lib/dapp/image/docker.rb +15 -8
  74. data/lib/dapp/image/stage.rb +10 -12
  75. data/lib/dapp/lock/base.rb +44 -0
  76. data/lib/dapp/lock/error.rb +14 -0
  77. data/lib/dapp/lock/file.rb +33 -0
  78. data/lib/dapp/prctl.rb +22 -0
  79. data/lib/dapp/project.rb +75 -0
  80. data/lib/dapp/project/command/bp.rb +24 -0
  81. data/lib/dapp/project/command/build.rb +21 -0
  82. data/lib/dapp/project/command/cleanup.rb +24 -0
  83. data/lib/dapp/project/command/common.rb +51 -0
  84. data/lib/dapp/project/command/list.rb +14 -0
  85. data/lib/dapp/project/command/push.rb +21 -0
  86. data/lib/dapp/project/command/run.rb +15 -0
  87. data/lib/dapp/project/command/spush.rb +17 -0
  88. data/lib/dapp/project/command/stages_cleanup.rb +70 -0
  89. data/lib/dapp/project/command/stages_flush.rb +18 -0
  90. data/lib/dapp/project/dappfile.rb +70 -0
  91. data/lib/dapp/project/lock.rb +27 -0
  92. data/lib/dapp/project/logging/base.rb +107 -0
  93. data/lib/dapp/project/logging/process.rb +104 -0
  94. data/lib/dapp/project/paint.rb +16 -0
  95. data/lib/dapp/project/ssh_agent.rb +77 -0
  96. data/lib/dapp/version.rb +1 -1
  97. metadata +74 -27
  98. data/lib/dapp/application/deps/gitartifact.rb +0 -39
  99. data/lib/dapp/application/logging.rb +0 -120
  100. data/lib/dapp/application/system_shellout.rb +0 -63
  101. data/lib/dapp/build/stage/chef_cookbooks.rb +0 -33
  102. data/lib/dapp/build/stage/infra_setup.rb +0 -27
  103. data/lib/dapp/build/stage/install.rb +0 -27
  104. data/lib/dapp/build/stage/setup.rb +0 -27
  105. data/lib/dapp/build/stage/source_1.rb +0 -21
  106. data/lib/dapp/build/stage/source_1_dependencies.rb +0 -27
  107. data/lib/dapp/build/stage/source_2.rb +0 -17
  108. data/lib/dapp/build/stage/source_2_dependencies.rb +0 -17
  109. data/lib/dapp/build/stage/source_3.rb +0 -17
  110. data/lib/dapp/build/stage/source_3_dependencies.rb +0 -23
  111. data/lib/dapp/build/stage/source_4.rb +0 -21
  112. data/lib/dapp/build/stage/source_4_dependencies.rb +0 -27
  113. data/lib/dapp/cli/smartpush.rb +0 -19
  114. data/lib/dapp/controller.rb +0 -119
  115. data/lib/dapp/error/controller.rb +0 -6
  116. data/lib/dapp/helper/log.rb +0 -73
@@ -0,0 +1,24 @@
1
+ module Dapp
2
+ # Project
3
+ class Project
4
+ # Command
5
+ module Command
6
+ # Build
7
+ module Bp
8
+ def bp(repo)
9
+ bp_step(:build)
10
+ bp_step(:push, repo)
11
+ bp_step(:stages_cleanup, repo)
12
+ bp_step(:cleanup)
13
+ end
14
+
15
+ def bp_step(step, *args)
16
+ log_step(step)
17
+ with_log_indent do
18
+ public_send(step, *args)
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end # Project
24
+ end # Dapp
@@ -0,0 +1,21 @@
1
+ module Dapp
2
+ # Project
3
+ class Project
4
+ # Command
5
+ module Command
6
+ # Build
7
+ module Build
8
+ def build
9
+ setup_ssh_agent
10
+
11
+ build_configs.each do |config|
12
+ log_step(config._name)
13
+ with_log_indent do
14
+ Application.new(config: config, project: self).build!
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end # Project
21
+ end # Dapp
@@ -0,0 +1,24 @@
1
+ module Dapp
2
+ # Project
3
+ class Project
4
+ # Command
5
+ module Command
6
+ # Cleanup
7
+ module Cleanup
8
+ def cleanup
9
+ build_configs.map(&:_basename).uniq.each do |basename|
10
+ lock("#{basename}.images") do
11
+ log(basename)
12
+ containers_flush(basename)
13
+ remove_images([
14
+ 'docker images',
15
+ %(--format '{{if ne "#{stage_cache(basename)}" .Repository }}{{.ID}}{{ end }}'),
16
+ %(-f "label=dapp=#{stage_dapp_label(basename)}")
17
+ ].join(' ')) # FIXME: negative filter is not currently supported by the Docker CLI
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end # Project
24
+ end # Dapp
@@ -0,0 +1,51 @@
1
+ module Dapp
2
+ # Project
3
+ class Project
4
+ # Command
5
+ module Command
6
+ # Common
7
+ module Common
8
+ protected
9
+
10
+ def containers_flush(basename)
11
+ remove_containers(%(docker ps -a -f "label=dapp" -f "name=#{container_name(basename)}" -q), force: true)
12
+ end
13
+
14
+ def remove_images(images_query, force: false)
15
+ force_option = force ? ' -f' : ''
16
+ with_subquery(images_query) { |ids| run_command(%(docker rmi#{force_option} #{ids.join(' ')})) }
17
+ end
18
+
19
+ def remove_containers(containers_query, force: false)
20
+ force_option = force ? ' -f' : ''
21
+ with_subquery(containers_query) { |ids| run_command(%(docker rm#{force_option} #{ids.join(' ')})) }
22
+ end
23
+
24
+ def with_subquery(query)
25
+ return if (res = shellout!(query).stdout.strip.lines.map(&:strip)).empty?
26
+ yield(res)
27
+ end
28
+
29
+ def run_command(cmd)
30
+ if @cli_options[:dry_run] # FIXME
31
+ puts cmd
32
+ else
33
+ shellout!(cmd)
34
+ end
35
+ end
36
+
37
+ def stage_cache(basename)
38
+ cache_format % { application_name: basename }
39
+ end
40
+
41
+ def stage_dapp_label(basename)
42
+ stage_dapp_label_format % { application_name: basename }
43
+ end
44
+
45
+ def container_name(basename)
46
+ basename
47
+ end
48
+ end
49
+ end
50
+ end # Project
51
+ end # Dapp
@@ -0,0 +1,14 @@
1
+ module Dapp
2
+ # Project
3
+ class Project
4
+ # Command
5
+ module Command
6
+ # List
7
+ module List
8
+ def list
9
+ build_configs.each { |config| puts config._name }
10
+ end
11
+ end
12
+ end
13
+ end # Project
14
+ end # Dapp
@@ -0,0 +1,21 @@
1
+ module Dapp
2
+ # Project
3
+ class Project
4
+ # Command
5
+ module Command
6
+ # Push
7
+ module Push
8
+ def push(repo)
9
+ build_configs.each do |config|
10
+ log_step(config._name)
11
+ with_log_indent do
12
+ Application.new(config: config, project: self, ignore_git_fetch: true, should_be_built: true).tap do |app|
13
+ app.export!(repo, format: '%{repo}:%{application_name}-%{tag}')
14
+ end
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end # Project
21
+ end # Dapp
@@ -0,0 +1,15 @@
1
+ module Dapp
2
+ # Project
3
+ class Project
4
+ # Command
5
+ module Command
6
+ # Run
7
+ module Run
8
+ def run(docker_options, command)
9
+ raise Error::Project, code: :run_command_unexpected_apps_number unless build_configs.one?
10
+ Application.new(config: build_configs.first, project: self, ignore_git_fetch: true, should_be_built: true).run(docker_options, command)
11
+ end
12
+ end
13
+ end
14
+ end # Project
15
+ end # Dapp
@@ -0,0 +1,17 @@
1
+ module Dapp
2
+ # Project
3
+ class Project
4
+ # Command
5
+ module Command
6
+ # Spush
7
+ module Spush
8
+ def spush(repo)
9
+ raise Error::Project, code: :spush_command_unexpected_apps_number unless build_configs.one?
10
+ Application.new(config: build_configs.first, project: self, ignore_git_fetch: true, should_be_built: true).tap do |app|
11
+ app.export!(repo, format: '%{repo}:%{tag}')
12
+ end
13
+ end
14
+ end
15
+ end
16
+ end # Project
17
+ end # Dapp
@@ -0,0 +1,70 @@
1
+ module Dapp
2
+ # Project
3
+ class Project
4
+ # Command
5
+ module Command
6
+ # StagesCleanup
7
+ module StagesCleanup
8
+ def stages_cleanup(repo)
9
+ repo_applications = repo_applications(repo)
10
+ build_configs.map(&:_basename).uniq.each do |basename|
11
+ log(basename)
12
+ containers_flush(basename)
13
+ apps, stages = project_images(basename).partition { |_, image_id| repo_applications.values.include?(image_id) }
14
+ apps, stages = apps.to_h, stages.to_h
15
+ apps.each { |_, aiid| stages = clear_stages(aiid, stages) }
16
+ run_command(%(docker rmi #{stages.keys.join(' ')})) unless stages.keys.empty?
17
+ end
18
+ end
19
+
20
+ def clear_stages(image_id, stages)
21
+ if image_exist?(image_id)
22
+ image_dapp_artifacts_label(image_id).each { |aiid| clear_stages(aiid, stages) }
23
+ iid = image_id
24
+ loop do
25
+ stages.delete_if { |_, siid| siid == iid }
26
+ break if (iid = image_parent(iid)).empty?
27
+ end
28
+ else
29
+ stages.delete_if { |_, siid| siid == iid }
30
+ end
31
+ stages
32
+ end
33
+
34
+ protected
35
+
36
+ def registry(repo)
37
+ @registry ||= DockerRegistry.new(repo)
38
+ end
39
+
40
+ def repo_applications(repo)
41
+ @repo_apps ||= begin
42
+ registry = registry(repo)
43
+ registry.tags.select { |tag| !tag.start_with?('dappstage') }.map { |tag| [tag, registry.image_id_by_tag(tag)] }.to_h
44
+ end
45
+ end
46
+
47
+ def project_images(basename)
48
+ shellout!(%(docker images --format "{{.Repository}}:{{.Tag}};{{.ID}}" --no-trunc #{stage_cache(basename)})).stdout.lines.map do |line|
49
+ line.strip.split(';')
50
+ end.to_h
51
+ end
52
+
53
+ def image_parent(image_id)
54
+ shellout!(%(docker inspect -f {{.Parent}} #{image_id})).stdout.strip
55
+ end
56
+
57
+ def image_dapp_artifacts_label(image_id)
58
+ Image::Docker.image_config_option(image_id: image_id, option: 'labels').select { |k, _v| k.start_with?('dapp-artifact') }.values
59
+ end
60
+
61
+ def image_exist?(image_id)
62
+ shellout!(%(docker inspect #{image_id}))
63
+ true
64
+ rescue Error::Shellout
65
+ false
66
+ end
67
+ end
68
+ end
69
+ end # Project
70
+ end # Dapp
@@ -0,0 +1,18 @@
1
+ module Dapp
2
+ # Project
3
+ class Project
4
+ # Command
5
+ module Command
6
+ # StagesFlush
7
+ module StagesFlush
8
+ def stages_flush
9
+ build_configs.map(&:_basename).uniq.each do |basename|
10
+ log(basename)
11
+ containers_flush(basename)
12
+ remove_images(%(docker images --format="{{.Repository}}:{{.Tag}}" #{stage_cache(basename)}))
13
+ end
14
+ end
15
+ end
16
+ end
17
+ end # Project
18
+ end # Dapp
@@ -0,0 +1,70 @@
1
+ module Dapp
2
+ # Project
3
+ class Project
4
+ # Dappfile
5
+ module Dappfile
6
+ def build_configs
7
+ @configs ||= begin
8
+ dappfiles.map { |dappfile| apps(dappfile, app_filters: apps_patterns) }.flatten.tap do |apps|
9
+ raise Error::Project, code: :no_such_app, data: { apps_patterns: apps_patterns.join(', ') } if apps.empty?
10
+ end
11
+ end
12
+ end
13
+
14
+ def dappfiles
15
+ if File.exist?(dappfile_path) then [dappfile_path]
16
+ elsif !dapps_dappfiles_pathes.empty? then dapps_dappfiles_pathes
17
+ elsif (dappfile_path = search_up('Dappfile')) then [dappfile_path]
18
+ else raise Error::Project, code: :dappfile_not_found
19
+ end
20
+ end
21
+
22
+ def dappfile_path
23
+ File.join [cli_options[:dir], 'Dappfile'].compact
24
+ end
25
+
26
+ def dapps_dappfiles_pathes
27
+ path = []
28
+ path << cli_options[:dir]
29
+ path << '.dapps' unless File.basename(work_dir) == '.dapps'
30
+ path << '*'
31
+ path << 'Dappfile'
32
+ Dir.glob(File.join(path.compact))
33
+ end
34
+
35
+ def search_up(file)
36
+ cdir = Pathname(work_dir)
37
+ loop do
38
+ if (path = cdir.join(file)).exist?
39
+ return path.to_s
40
+ end
41
+ break if (cdir = cdir.parent).root?
42
+ end
43
+ end
44
+
45
+ def work_dir
46
+ File.expand_path(cli_options[:dir] || Dir.pwd)
47
+ end
48
+
49
+ def expand_path(path, number = 1)
50
+ path = File.expand_path(path)
51
+ number.times.each { path = File.dirname(path) }
52
+ path
53
+ end
54
+
55
+ def apps(dappfile_path, app_filters:)
56
+ config = Config::Main.new(dappfile_path: dappfile_path, project: self) do |conf|
57
+ begin
58
+ conf.instance_eval File.read(dappfile_path), dappfile_path
59
+ rescue SyntaxError, StandardError => e
60
+ backtrace = e.backtrace.find { |line| line.start_with?(dappfile_path) }
61
+ message = e.is_a?(NoMethodError) ? e.message[/.*(?= for)/] : e.message
62
+ message = "#{backtrace[/.*(?=:in)/]}: #{message}" if backtrace
63
+ raise Error::Dappfile, code: :incorrect, data: { error: e.class.name, message: message }
64
+ end
65
+ end
66
+ config._apps.select { |app| app_filters.any? { |pattern| File.fnmatch(pattern, app._name) } }
67
+ end
68
+ end # Dappfile
69
+ end # Project
70
+ end # Dapp
@@ -0,0 +1,27 @@
1
+ module Dapp
2
+ # Project
3
+ class Project
4
+ # Lock
5
+ module Lock
6
+ def lock_path
7
+ build_path.join('locks')
8
+ end
9
+
10
+ def lock(name, *args, default_timeout: 300, **kwargs, &blk)
11
+ timeout = cli_options[:lock_timeout] || default_timeout
12
+
13
+ ::Dapp::Lock::File.new(
14
+ lock_path, name,
15
+ timeout: timeout,
16
+ on_wait: lambda do |&blk|
17
+ log_secondary_process(
18
+ t(code: 'process.waiting_resouce_lock', data: { name: name }),
19
+ short: true,
20
+ &blk
21
+ )
22
+ end
23
+ ).synchronize(*args, **kwargs, &blk)
24
+ end
25
+ end # Lock
26
+ end # Project
27
+ end # Dapp
@@ -0,0 +1,107 @@
1
+ module Dapp
2
+ # Project
3
+ class Project
4
+ # Logging
5
+ module Logging
6
+ # Base
7
+ module Base
8
+ def log_quiet?
9
+ cli_options[:log_quiet]
10
+ end
11
+
12
+ def log_time?
13
+ cli_options[:log_time]
14
+ end
15
+
16
+ def log_verbose?
17
+ cli_options[:log_verbose]
18
+ end
19
+
20
+ def dry_run?
21
+ cli_options[:dry_run]
22
+ end
23
+
24
+ def log_info(*args, **kwargs)
25
+ kwargs[:style] = :info
26
+ log(*args, **kwargs)
27
+ end
28
+
29
+ def log_step(*args, **kwargs)
30
+ kwargs[:style] = :step
31
+ log(*args, **kwargs)
32
+ end
33
+
34
+ def log_secondary(*args, **kwargs)
35
+ kwargs[:style] = :secondary
36
+ log(*args, **kwargs)
37
+ end
38
+
39
+ def log_warning(*args, **kwargs)
40
+ kwargs[:style] = :warning
41
+ log(*args, **kwargs)
42
+ end
43
+
44
+ def log(message = '', desc: nil, inline: false, **kwargs)
45
+ return if log_quiet?
46
+ unless desc.nil?
47
+ (desc[:data] ||= {})[:msg] = message
48
+ message = t(**desc)
49
+ end
50
+ print "#{log_format_string(message, **kwargs)}#{"\n" unless inline}"
51
+ end
52
+
53
+ def log_time
54
+ self.class.log_time
55
+ end
56
+
57
+ def log_format_string(str, time: true, indent: true, style: nil)
58
+ str.to_s.lines.map do |line|
59
+ line = paint_string(line, style) if style
60
+ "#{log_time if time && log_time?}#{indent ? (log_indent + line) : line}"
61
+ end.join
62
+ end
63
+
64
+ def log_with_indent(message = '', **kwargs)
65
+ with_log_indent do
66
+ log(message, **kwargs)
67
+ end
68
+ end
69
+
70
+ def with_log_indent(with = true)
71
+ log_indent_next if with
72
+ yield
73
+ ensure
74
+ log_indent_prev if with
75
+ end
76
+
77
+ def log_indent
78
+ ' ' * 2 * log_indent_size
79
+ end
80
+
81
+ def log_indent_next
82
+ self.log_indent_size += 1
83
+ end
84
+
85
+ def log_indent_prev
86
+ if self.log_indent_size <= 0
87
+ self.log_indent_size = 0
88
+ else
89
+ self.log_indent_size -= 1
90
+ end
91
+ end
92
+
93
+ def self.log_time
94
+ "#{DateTime.now.strftime('%Y-%m-%dT%T%z')} "
95
+ end
96
+
97
+ protected
98
+
99
+ attr_writer :log_indent_size
100
+
101
+ def log_indent_size
102
+ @log_indent_size ||= 0
103
+ end
104
+ end
105
+ end # Logging
106
+ end # Project
107
+ end # Dapp