rbld 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/cli/bin/rbld +35 -0
- data/cli/lib/bootstrap/re-build-bootstrap-utils +30 -0
- data/cli/lib/bootstrap/re-build-entry-point +75 -0
- data/cli/lib/bootstrap/re-build-env-prepare +52 -0
- data/cli/lib/bootstrap/rebuild.rc +6 -0
- data/cli/lib/commands/rbld_checkout.rb +14 -0
- data/cli/lib/commands/rbld_commit.rb +36 -0
- data/cli/lib/commands/rbld_create.rb +50 -0
- data/cli/lib/commands/rbld_deploy.rb +16 -0
- data/cli/lib/commands/rbld_list.rb +12 -0
- data/cli/lib/commands/rbld_load.rb +25 -0
- data/cli/lib/commands/rbld_modify.rb +22 -0
- data/cli/lib/commands/rbld_publish.rb +15 -0
- data/cli/lib/commands/rbld_rm.rb +14 -0
- data/cli/lib/commands/rbld_run.rb +24 -0
- data/cli/lib/commands/rbld_save.rb +28 -0
- data/cli/lib/commands/rbld_search.rb +13 -0
- data/cli/lib/commands/rbld_status.rb +12 -0
- data/cli/lib/rbld_commands.rb +207 -0
- data/cli/lib/rbld_config.rb +20 -0
- data/cli/lib/rbld_engine.rb +710 -0
- data/cli/lib/rbld_log.rb +26 -0
- data/cli/lib/rbld_print.rb +66 -0
- data/cli/lib/rbld_registry.rb +100 -0
- data/cli/lib/rbld_utils.rb +54 -0
- data/tools/rebuild-conf/Rakefile +19 -0
- metadata +294 -0
@@ -0,0 +1,28 @@
|
|
1
|
+
module Rebuild::CLI
|
2
|
+
class RbldSaveCommand < Command
|
3
|
+
private
|
4
|
+
|
5
|
+
def default_file(env)
|
6
|
+
"#{env.name}-#{env.tag}.rbld"
|
7
|
+
end
|
8
|
+
|
9
|
+
public
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
@usage = "save [OPTIONS] [ENVIRONMENT] [FILE]"
|
13
|
+
@description = "Save local environment to file"
|
14
|
+
end
|
15
|
+
|
16
|
+
def run(parameters)
|
17
|
+
env, file = parameters
|
18
|
+
env = Environment.new( env )
|
19
|
+
file = default_file( env ) if !file or file.empty?
|
20
|
+
rbld_log.info("Going to save #{env} to #{file}")
|
21
|
+
|
22
|
+
warn_if_modified( env, 'saving' )
|
23
|
+
engine_api.save(env, file)
|
24
|
+
|
25
|
+
rbld_print.progress "Successfully saved environment #{env} to #{file}\n"
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module Rebuild::CLI
|
2
|
+
class RbldSearchCommand < Command
|
3
|
+
def initialize
|
4
|
+
@usage = "search [OPTIONS] [NAME[:TAG]|PREFIX]"
|
5
|
+
@description = "Search remote registry for published environments"
|
6
|
+
end
|
7
|
+
|
8
|
+
def run(parameters)
|
9
|
+
env = Environment.new(parameters[0], allow_empty: true)
|
10
|
+
print_names( engine_api.search( env ) )
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
module Rebuild::CLI
|
2
|
+
class RbldStatusCommand < Command
|
3
|
+
def initialize
|
4
|
+
@usage = "status [OPTIONS]"
|
5
|
+
@description = "List modified environments"
|
6
|
+
end
|
7
|
+
|
8
|
+
def run(parameters)
|
9
|
+
print_names( engine_api.environments.select( &:modified? ), 'modified: ' )
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,207 @@
|
|
1
|
+
require_relative 'rbld_log'
|
2
|
+
require_relative 'rbld_utils'
|
3
|
+
require_relative 'rbld_engine'
|
4
|
+
|
5
|
+
module Rebuild::CLI
|
6
|
+
extend Rebuild::Utils::Errors
|
7
|
+
|
8
|
+
rebuild_errors \
|
9
|
+
EnvironmentNameEmpty: 'Environment name not specified',
|
10
|
+
EnvironmentNameWithoutTagExpected: 'Environment tag must not be specified',
|
11
|
+
EnvironmentNameError: 'Invalid %s, it may contain a-z, A-Z, 0-9, - and _ characters only',
|
12
|
+
HandlerClassNameError: '%s'
|
13
|
+
|
14
|
+
class Environment
|
15
|
+
def initialize(env, opts = {})
|
16
|
+
if env.respond_to?( :name ) && env.respond_to?( :tag )
|
17
|
+
@name, @tag = env.name, env.tag
|
18
|
+
else
|
19
|
+
deduce_name_tag( env, opts )
|
20
|
+
validate_name_tag(opts)
|
21
|
+
end
|
22
|
+
@full = "#{@name}:#{@tag}"
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.validate_component( name, value )
|
26
|
+
raise EnvironmentNameError, "#{name} (#{value})" \
|
27
|
+
unless value.match( /^[[:alnum:]\_\-]*$/ )
|
28
|
+
end
|
29
|
+
|
30
|
+
def to_s
|
31
|
+
@full
|
32
|
+
end
|
33
|
+
|
34
|
+
attr_reader :name, :tag, :full
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def parse_name_tag(cli_param, opts)
|
39
|
+
if opts[:allow_empty] && (!cli_param || cli_param.empty?)
|
40
|
+
@name, @tag = '', ''
|
41
|
+
elsif !cli_param || cli_param.empty?
|
42
|
+
raise EnvironmentNameEmpty
|
43
|
+
else
|
44
|
+
@name, @tag = cli_param.match( /^([^:]*):?(.*)/ ).captures
|
45
|
+
@tag = '' if @name.empty?
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def deduce_name_tag(cli_param, opts)
|
50
|
+
parse_name_tag( cli_param, opts )
|
51
|
+
|
52
|
+
raise EnvironmentNameWithoutTagExpected \
|
53
|
+
if opts[:force_no_tag] && !@tag.empty?
|
54
|
+
|
55
|
+
@tag = 'initial' if @tag.empty? && !opts[:allow_empty]
|
56
|
+
end
|
57
|
+
|
58
|
+
def validate_name_tag(opts)
|
59
|
+
raise EnvironmentNameEmpty if @name.empty? && !opts[:allow_empty]
|
60
|
+
self.class.validate_component( "environment name", @name ) unless @name.empty?
|
61
|
+
self.class.validate_component( "environment tag", @tag ) unless @tag.empty?
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
class Commands
|
66
|
+
extend Enumerable
|
67
|
+
|
68
|
+
private
|
69
|
+
|
70
|
+
def self.deduce_cmd_name(handler_class)
|
71
|
+
match = handler_class.name.match(/Rbld(.*)Command/)
|
72
|
+
return nil unless match
|
73
|
+
match.captures[0].downcase
|
74
|
+
end
|
75
|
+
|
76
|
+
def self.handler!(command)
|
77
|
+
@handler_classes.each do |klass|
|
78
|
+
return klass.new if command == deduce_cmd_name( klass )
|
79
|
+
end
|
80
|
+
|
81
|
+
raise "Unknown command: #{command}"
|
82
|
+
end
|
83
|
+
|
84
|
+
@handler_classes = []
|
85
|
+
|
86
|
+
public
|
87
|
+
|
88
|
+
def self.register_handler_class(klass)
|
89
|
+
unless deduce_cmd_name( klass )
|
90
|
+
raise HandlerClassNameError, klass.name
|
91
|
+
end
|
92
|
+
|
93
|
+
@handler_classes << klass
|
94
|
+
end
|
95
|
+
|
96
|
+
def self.each
|
97
|
+
@handler_classes.each { |klass| yield( deduce_cmd_name( klass ) ) }
|
98
|
+
end
|
99
|
+
|
100
|
+
def self.usage(command)
|
101
|
+
handler!( command ).usage
|
102
|
+
end
|
103
|
+
|
104
|
+
def self.run(command, parameters)
|
105
|
+
handler = handler!( command )
|
106
|
+
handler.run( parameters )
|
107
|
+
handler.errno
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
class Command
|
112
|
+
|
113
|
+
private
|
114
|
+
|
115
|
+
def self.inherited( handler_class )
|
116
|
+
Commands.register_handler_class( handler_class )
|
117
|
+
rbld_log.info( "Command handler class #{handler_class} registered" )
|
118
|
+
end
|
119
|
+
|
120
|
+
def options_text
|
121
|
+
options = (@options || []) + [["-h, --help", "Print usage"]]
|
122
|
+
text = ""
|
123
|
+
options.each { |o| text << " #{o[0].ljust(30)}#{o[1]}\n" }
|
124
|
+
text
|
125
|
+
end
|
126
|
+
|
127
|
+
def replace_argv(parameters)
|
128
|
+
orig_argv = ARGV.clone
|
129
|
+
ARGV.clear
|
130
|
+
parameters.each { |x| ARGV << x }
|
131
|
+
yield
|
132
|
+
ARGV.clear
|
133
|
+
orig_argv.each { |x| ARGV << x }
|
134
|
+
end
|
135
|
+
|
136
|
+
def print_names(names, prefix = '')
|
137
|
+
strings = names.map { |n| Environment.new(n).full }
|
138
|
+
puts
|
139
|
+
strings.sort.each { |s| puts " #{prefix}#{s}"}
|
140
|
+
puts
|
141
|
+
end
|
142
|
+
|
143
|
+
def engine_api
|
144
|
+
@engine_api ||= Rebuild::Engine::API.new
|
145
|
+
@engine_api
|
146
|
+
end
|
147
|
+
|
148
|
+
def warn_if_modified(env, action)
|
149
|
+
rbld_print.warning "Environment is modified, #{action} original version" \
|
150
|
+
if engine_api.environments.select( &:modified? ).include?( env )
|
151
|
+
end
|
152
|
+
|
153
|
+
def get_cmdline_tail(parameters)
|
154
|
+
parameters.shift if parameters[0] == '--'
|
155
|
+
parameters
|
156
|
+
end
|
157
|
+
|
158
|
+
def format_usage_text
|
159
|
+
text = ""
|
160
|
+
if @usage.respond_to?(:each)
|
161
|
+
text << "\n"
|
162
|
+
@usage.each do |mode|
|
163
|
+
text << "\n rbld #{mode[:syntax]}\n\n" \
|
164
|
+
" #{mode[:description]}\n"
|
165
|
+
end
|
166
|
+
else
|
167
|
+
text << "rbld #{@usage}\n"
|
168
|
+
end
|
169
|
+
text
|
170
|
+
end
|
171
|
+
|
172
|
+
public
|
173
|
+
|
174
|
+
attr_reader :errno
|
175
|
+
|
176
|
+
def usage
|
177
|
+
puts <<END_USAGE
|
178
|
+
|
179
|
+
Usage: #{format_usage_text}
|
180
|
+
#{@description}
|
181
|
+
|
182
|
+
#{options_text}
|
183
|
+
END_USAGE
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
class Main
|
188
|
+
|
189
|
+
def self.usage
|
190
|
+
usage_text = <<USAGE
|
191
|
+
Usage:
|
192
|
+
rbld help Show this help screen
|
193
|
+
rbld help COMMAND Show help for COMMAND
|
194
|
+
rbld COMMAND [PARAMS] Run COMMAND with PARAMS
|
195
|
+
|
196
|
+
rebuild: Zero-dependency, reproducible build environments
|
197
|
+
|
198
|
+
Commands:
|
199
|
+
|
200
|
+
USAGE
|
201
|
+
|
202
|
+
Commands.sort.each { |cmd| usage_text << " #{cmd}\n"}
|
203
|
+
|
204
|
+
usage_text
|
205
|
+
end
|
206
|
+
end
|
207
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'parseconfig'
|
2
|
+
|
3
|
+
module Rebuild
|
4
|
+
class Config
|
5
|
+
def initialize()
|
6
|
+
cfg_file = File.join( Dir.home, '.rbld', 'rebuild.conf' )
|
7
|
+
|
8
|
+
if File.exist?( cfg_file )
|
9
|
+
cfg = ParseConfig.new( cfg_file )
|
10
|
+
rname = cfg['REMOTE_NAME']
|
11
|
+
@remote = rname ? cfg["REMOTE_#{rname}"] : nil
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def remote!
|
16
|
+
raise 'Remote not defined' unless @remote
|
17
|
+
@remote
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,710 @@
|
|
1
|
+
require 'docker'
|
2
|
+
require 'etc'
|
3
|
+
require 'thread'
|
4
|
+
require 'forwardable'
|
5
|
+
require_relative 'rbld_log'
|
6
|
+
require_relative 'rbld_config'
|
7
|
+
require_relative 'rbld_utils'
|
8
|
+
require_relative 'rbld_print'
|
9
|
+
require_relative 'rbld_registry'
|
10
|
+
|
11
|
+
module Rebuild::Engine
|
12
|
+
extend Rebuild::Utils::Errors
|
13
|
+
|
14
|
+
class NamedDockerImage
|
15
|
+
extend Forwardable
|
16
|
+
|
17
|
+
def_delegators :@api_obj, :id
|
18
|
+
attr_reader :api_obj
|
19
|
+
|
20
|
+
def initialize(name, api_obj)
|
21
|
+
@name, @api_obj = name, api_obj
|
22
|
+
end
|
23
|
+
|
24
|
+
def remove!
|
25
|
+
@api_obj.remove( name: @name.to_s )
|
26
|
+
end
|
27
|
+
|
28
|
+
def tag
|
29
|
+
@api_obj.tag( repo: @name.repo,
|
30
|
+
tag: @name.tag )
|
31
|
+
end
|
32
|
+
|
33
|
+
def identity
|
34
|
+
@name.to_s
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
class NamedDockerContainer
|
39
|
+
def initialize(name, api_obj)
|
40
|
+
@name, @api_obj = name, api_obj
|
41
|
+
end
|
42
|
+
|
43
|
+
def remove!
|
44
|
+
@api_obj.delete( force: true )
|
45
|
+
end
|
46
|
+
|
47
|
+
def flatten(img_name)
|
48
|
+
data_queue = Queue.new
|
49
|
+
new_img = nil
|
50
|
+
|
51
|
+
exporter = Thread.new do
|
52
|
+
@api_obj.export { |chunk| data_queue << chunk }
|
53
|
+
data_queue << ''
|
54
|
+
end
|
55
|
+
|
56
|
+
importer = Thread.new do
|
57
|
+
opts = commit_opts(img_name)
|
58
|
+
new_img = Docker::Image.import_stream(opts) { data_queue.pop }
|
59
|
+
end
|
60
|
+
|
61
|
+
exporter.join
|
62
|
+
importer.join
|
63
|
+
|
64
|
+
rbld_log.info("Created image #{new_img} from #{@api_obj}")
|
65
|
+
remove!
|
66
|
+
|
67
|
+
new_img
|
68
|
+
end
|
69
|
+
|
70
|
+
def commit(img_name)
|
71
|
+
if squash_needed?
|
72
|
+
flatten( img_name )
|
73
|
+
else
|
74
|
+
new_img = @api_obj.commit( commit_opts( img_name ) )
|
75
|
+
rbld_log.info("Created image #{new_img} from #{@api_obj}")
|
76
|
+
remove!
|
77
|
+
new_img
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
private
|
82
|
+
|
83
|
+
def squash_needed?
|
84
|
+
#Different FS backends have different limitations for
|
85
|
+
#maximum number of layers, 40 looks like small enough
|
86
|
+
#to be supported by all possible configurations
|
87
|
+
Docker::Image.get(@api_obj.info["ImageID"]).history.size >= 40
|
88
|
+
end
|
89
|
+
|
90
|
+
def commit_opts(img_name)
|
91
|
+
{repo: img_name.repo,
|
92
|
+
tag: img_name.tag,
|
93
|
+
changes: ["LABEL re-build-environment=true",
|
94
|
+
"ENTRYPOINT [\"/rebuild/re-build-entry-point\"]"]}
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
class NameFactory
|
99
|
+
def initialize(env)
|
100
|
+
@env = env
|
101
|
+
end
|
102
|
+
|
103
|
+
def identity
|
104
|
+
Rebuild::Utils::FullImageName.new("rbe-#{@env.name}", @env.tag)
|
105
|
+
end
|
106
|
+
|
107
|
+
def rerun
|
108
|
+
Rebuild::Utils::FullImageName.new("rbr-#{@env.name}-rt-#{@env.tag}", 'initial')
|
109
|
+
end
|
110
|
+
|
111
|
+
def running
|
112
|
+
container_name(:running)
|
113
|
+
end
|
114
|
+
|
115
|
+
def modified
|
116
|
+
container_name(:dirty)
|
117
|
+
end
|
118
|
+
|
119
|
+
def hostname
|
120
|
+
"#{@env.name}-#{@env.tag}"
|
121
|
+
end
|
122
|
+
|
123
|
+
def modified_hostname
|
124
|
+
"#{hostname}-M"
|
125
|
+
end
|
126
|
+
|
127
|
+
private
|
128
|
+
|
129
|
+
def container_name(type)
|
130
|
+
"rbe-#{type.to_s.chars.first}-#{@env.name}-rt-#{@env.tag}"
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
class Environment
|
135
|
+
def self.from_image(img_name, api_obj)
|
136
|
+
if match = img_name.match(/^rbe-(.*):(.*)/)
|
137
|
+
new( *match.captures, NamedDockerImage.new( img_name, api_obj ) )
|
138
|
+
else
|
139
|
+
nil
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
def attach_container(cont_name, api_obj)
|
144
|
+
try_attach_container(:dirty, cont_name, api_obj) ||
|
145
|
+
try_attach_container(:running, cont_name, api_obj)
|
146
|
+
end
|
147
|
+
|
148
|
+
def attach_rerun_image(img_name, api_obj)
|
149
|
+
match = img_name.match(/^rbr-(.*)-rt-(.*):initial/)
|
150
|
+
if my_object?( match )
|
151
|
+
@rerun_img = NamedDockerImage.new( img_name, api_obj )
|
152
|
+
true
|
153
|
+
else
|
154
|
+
false
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
def modified?
|
159
|
+
!@cont[:dirty].nil?
|
160
|
+
end
|
161
|
+
|
162
|
+
def execution_container
|
163
|
+
@cont[:running]
|
164
|
+
end
|
165
|
+
|
166
|
+
def modification_container
|
167
|
+
@cont[:dirty]
|
168
|
+
end
|
169
|
+
|
170
|
+
def ==(other)
|
171
|
+
(other.name == @name) && (other.tag == @tag)
|
172
|
+
end
|
173
|
+
|
174
|
+
attr_reader :name, :tag, :img, :rerun_img
|
175
|
+
|
176
|
+
private
|
177
|
+
|
178
|
+
def initialize(name, tag, img)
|
179
|
+
@name, @tag, @img, @cont = name, tag, img, {}
|
180
|
+
end
|
181
|
+
|
182
|
+
def my_object?(match)
|
183
|
+
match && (match[1] == @name) && (match[2] == @tag)
|
184
|
+
end
|
185
|
+
|
186
|
+
def try_attach_container(type, cont_name, api_obj)
|
187
|
+
match = cont_name.match(/^\/rbe-#{type.to_s.chars.first}-(.*)-rt-(.*)/)
|
188
|
+
if my_object?( match )
|
189
|
+
@cont[type] = NamedDockerContainer.new( cont_name, api_obj )
|
190
|
+
true
|
191
|
+
else
|
192
|
+
false
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
private_class_method :new
|
197
|
+
end
|
198
|
+
|
199
|
+
class PresentEnvironments
|
200
|
+
include Enumerable
|
201
|
+
extend Forwardable
|
202
|
+
|
203
|
+
def initialize(api_module = Docker)
|
204
|
+
@api_module = api_module
|
205
|
+
refresh!
|
206
|
+
end
|
207
|
+
|
208
|
+
def refresh!
|
209
|
+
cache_images
|
210
|
+
attach_containers
|
211
|
+
attach_rerun_images
|
212
|
+
end
|
213
|
+
|
214
|
+
def_delegators :@all, :each
|
215
|
+
attr_reader :all
|
216
|
+
|
217
|
+
def dangling
|
218
|
+
rbld_images(dangling: ['true'])
|
219
|
+
end
|
220
|
+
|
221
|
+
def get(name)
|
222
|
+
find { |e| e == name }
|
223
|
+
end
|
224
|
+
|
225
|
+
private
|
226
|
+
|
227
|
+
def rbld_obj_filter
|
228
|
+
{ label: ["re-build-environment=true"] }
|
229
|
+
end
|
230
|
+
|
231
|
+
def rbld_images(filters = nil)
|
232
|
+
filters = rbld_obj_filter.merge( filters || {} )
|
233
|
+
@api_module::Image.all( :filters => filters.to_json )
|
234
|
+
end
|
235
|
+
|
236
|
+
def rbld_containers
|
237
|
+
@api_module::Container.all( all: true, filters: rbld_obj_filter.to_json )
|
238
|
+
end
|
239
|
+
|
240
|
+
def cache_images
|
241
|
+
@all = []
|
242
|
+
rbld_images.each do |img|
|
243
|
+
rbld_log.debug("Found docker image #{img}")
|
244
|
+
img.info['RepoTags'].each { |tag| @all << Environment.from_image( tag, img ) }
|
245
|
+
end
|
246
|
+
@all.compact!
|
247
|
+
end
|
248
|
+
|
249
|
+
def attach_containers
|
250
|
+
rbld_containers.each do |cont|
|
251
|
+
rbld_log.debug("Found docker container #{cont}")
|
252
|
+
cont.info['Names'].each do |name|
|
253
|
+
@all.find { |e| e.attach_container( name, cont ) }
|
254
|
+
end
|
255
|
+
end
|
256
|
+
end
|
257
|
+
|
258
|
+
def attach_rerun_images
|
259
|
+
rbld_images.each do |img|
|
260
|
+
rbld_log.debug("Found docker image #{img}")
|
261
|
+
img.info['RepoTags'].each do |tag|
|
262
|
+
@all.find { |e| e.attach_rerun_image( tag, img ) }
|
263
|
+
end
|
264
|
+
end
|
265
|
+
end
|
266
|
+
end
|
267
|
+
|
268
|
+
rebuild_errors \
|
269
|
+
UnsupportedDockerService: 'Unsupported docker service: %s',
|
270
|
+
EnvironmentIsModified: 'Environment is modified, commit or checkout first',
|
271
|
+
EnvironmentNotKnown: 'Unknown environment %s',
|
272
|
+
NoChangesToCommit: 'No changes to commit for %s',
|
273
|
+
EnvironmentLoadFailure: 'Failed to load environment from %s',
|
274
|
+
EnvironmentSaveFailure: 'Failed to save environment %s to %s',
|
275
|
+
EnvironmentDeploymentFailure: 'Failed to deploy from %s',
|
276
|
+
EnvironmentAlreadyExists: 'Environment %s already exists',
|
277
|
+
EnvironmentNotFoundInTheRegistry: 'Environment %s does not exist in the registry',
|
278
|
+
RegistrySearchFailed: 'Failed to search in %s',
|
279
|
+
EnvironmentPublishCollision: 'Environment %s already published',
|
280
|
+
EnvironmentPublishFailure: 'Failed to publish on %s',
|
281
|
+
EnvironmentCreateFailure: 'Failed to create %s'
|
282
|
+
|
283
|
+
class EnvironmentFile
|
284
|
+
def initialize(filename, docker_api = Docker)
|
285
|
+
@filename, @docker_api = filename, docker_api
|
286
|
+
end
|
287
|
+
|
288
|
+
def load!
|
289
|
+
begin
|
290
|
+
with_gzip_reader { |gz| Docker::Image.load(gz) }
|
291
|
+
rescue => msg
|
292
|
+
rbld_print.trace( msg )
|
293
|
+
raise EnvironmentLoadFailure, @filename
|
294
|
+
end
|
295
|
+
end
|
296
|
+
|
297
|
+
def save!(name, identity)
|
298
|
+
begin
|
299
|
+
with_gzip_writer do |gz|
|
300
|
+
Docker::Image.save_stream( identity ) { |chunk| gz.write chunk }
|
301
|
+
end
|
302
|
+
rescue => msg
|
303
|
+
rbld_print.trace( msg )
|
304
|
+
raise EnvironmentSaveFailure, [name, @filename]
|
305
|
+
end
|
306
|
+
end
|
307
|
+
|
308
|
+
private
|
309
|
+
|
310
|
+
def with_gzip_writer
|
311
|
+
begin
|
312
|
+
File.open(@filename, 'w') do |f|
|
313
|
+
gz = Zlib::GzipWriter.new(f)
|
314
|
+
begin
|
315
|
+
yield gz
|
316
|
+
ensure
|
317
|
+
gz.close
|
318
|
+
end
|
319
|
+
end
|
320
|
+
rescue
|
321
|
+
FileUtils::safe_unlink( @filename )
|
322
|
+
raise
|
323
|
+
end
|
324
|
+
end
|
325
|
+
|
326
|
+
def with_gzip_reader
|
327
|
+
Zlib::GzipReader.open( @filename ) do |gz|
|
328
|
+
begin
|
329
|
+
yield gz
|
330
|
+
ensure
|
331
|
+
gz.close
|
332
|
+
end
|
333
|
+
end
|
334
|
+
end
|
335
|
+
end
|
336
|
+
|
337
|
+
class DockerContext
|
338
|
+
def self.from_file(file)
|
339
|
+
base = %Q{
|
340
|
+
FROM scratch
|
341
|
+
ADD #{file} /
|
342
|
+
}
|
343
|
+
|
344
|
+
new( base, file )
|
345
|
+
end
|
346
|
+
|
347
|
+
def self.from_image(img)
|
348
|
+
base = %Q{
|
349
|
+
FROM #{img}
|
350
|
+
}
|
351
|
+
|
352
|
+
new( base )
|
353
|
+
end
|
354
|
+
|
355
|
+
def initialize(base, basefile = nil)
|
356
|
+
@base, @basefile = base, basefile
|
357
|
+
end
|
358
|
+
|
359
|
+
def prepare
|
360
|
+
tarfile_name = Dir::Tmpname.create('rbldctx') {}
|
361
|
+
|
362
|
+
rbld_log.info("Storing context in #{tarfile_name}")
|
363
|
+
|
364
|
+
File.open(tarfile_name, 'wb+') do |tarfile|
|
365
|
+
Gem::Package::TarWriter.new( tarfile ) do |tar|
|
366
|
+
|
367
|
+
files = bootstrap_file_pathnames
|
368
|
+
files << @basefile unless @basefile.nil?
|
369
|
+
|
370
|
+
files.each do |file_name|
|
371
|
+
tar.add_file(File.basename(file_name), 0640) do |t|
|
372
|
+
IO.copy_stream(file_name, t)
|
373
|
+
end
|
374
|
+
end
|
375
|
+
|
376
|
+
tar.add_file('Dockerfile', 0640) do |t|
|
377
|
+
t.write( dockerfile )
|
378
|
+
end
|
379
|
+
|
380
|
+
end
|
381
|
+
end
|
382
|
+
|
383
|
+
File.open(tarfile_name, 'r') { |f| yield f }
|
384
|
+
ensure
|
385
|
+
FileUtils::rm_f( tarfile_name )
|
386
|
+
end
|
387
|
+
|
388
|
+
private
|
389
|
+
|
390
|
+
def bootstrap_files
|
391
|
+
["re-build-bootstrap-utils",
|
392
|
+
"re-build-entry-point",
|
393
|
+
"re-build-env-prepare",
|
394
|
+
"rebuild.rc"]
|
395
|
+
end
|
396
|
+
|
397
|
+
def bootstrap_file_pathnames
|
398
|
+
src_path = File.join( File.dirname( __FILE__ ), "bootstrap" )
|
399
|
+
src_path = File.expand_path( src_path )
|
400
|
+
bootstrap_files.map { |f| File.join( src_path, f ) }
|
401
|
+
end
|
402
|
+
|
403
|
+
def dockerfile
|
404
|
+
# sync after chmod is needed because of an AuFS problem described in:
|
405
|
+
# https://github.com/docker/docker/issues/9547
|
406
|
+
%Q{
|
407
|
+
#{@base}
|
408
|
+
LABEL re-build-environment=true
|
409
|
+
COPY #{bootstrap_files.join(' ')} /rebuild/
|
410
|
+
RUN chown root:root \
|
411
|
+
/rebuild/re-build-env-prepare \
|
412
|
+
/rebuild/re-build-bootstrap-utils \
|
413
|
+
/rebuild/rebuild.rc && \
|
414
|
+
chmod 700 \
|
415
|
+
/rebuild/re-build-entry-point \
|
416
|
+
/rebuild/re-build-env-prepare && \
|
417
|
+
sync && \
|
418
|
+
chmod 644 \
|
419
|
+
/rebuild/rebuild.rc \
|
420
|
+
/rebuild/re-build-bootstrap-utils && \
|
421
|
+
sync && \
|
422
|
+
/rebuild/re-build-env-prepare
|
423
|
+
ENTRYPOINT ["/rebuild/re-build-entry-point"]
|
424
|
+
}.squeeze(' ')
|
425
|
+
end
|
426
|
+
|
427
|
+
private_class_method :new
|
428
|
+
end
|
429
|
+
|
430
|
+
class API
|
431
|
+
extend Forwardable
|
432
|
+
|
433
|
+
def initialize(docker_api = Docker, cfg = Rebuild::Config.new)
|
434
|
+
@docker_api, @cfg = docker_api, cfg
|
435
|
+
|
436
|
+
tweak_excon
|
437
|
+
check_connectivity
|
438
|
+
@cache = PresentEnvironments.new
|
439
|
+
end
|
440
|
+
|
441
|
+
def remove!(env_name)
|
442
|
+
env = unmodified_env( env_name )
|
443
|
+
env.img.remove!
|
444
|
+
@cache.refresh!
|
445
|
+
end
|
446
|
+
|
447
|
+
def load!(filename)
|
448
|
+
EnvironmentFile.new( filename ).load!
|
449
|
+
# If image with the same name but another
|
450
|
+
# ID existed before load it becomes dangling
|
451
|
+
# and should be ditched
|
452
|
+
@cache.dangling.each(&:remove)
|
453
|
+
@cache.refresh!
|
454
|
+
end
|
455
|
+
|
456
|
+
def save(env_name, filename)
|
457
|
+
env = existing_env( env_name )
|
458
|
+
EnvironmentFile.new( filename ).save!(env_name, env.img.identity)
|
459
|
+
end
|
460
|
+
|
461
|
+
def search(env_name)
|
462
|
+
rbld_print.progress "Searching in #{@cfg.remote!}..."
|
463
|
+
|
464
|
+
begin
|
465
|
+
registry.search( env_name.name, env_name.tag )
|
466
|
+
rescue => msg
|
467
|
+
rbld_print.trace( msg )
|
468
|
+
raise RegistrySearchFailed, @cfg.remote!
|
469
|
+
end
|
470
|
+
end
|
471
|
+
|
472
|
+
def deploy!(env_name)
|
473
|
+
nonexisting_env(env_name)
|
474
|
+
|
475
|
+
raise EnvironmentNotFoundInTheRegistry, env_name.full \
|
476
|
+
if registry.search( env_name.name, env_name.tag ).empty?
|
477
|
+
|
478
|
+
rbld_print.progress "Deploying from #{@cfg.remote!}..."
|
479
|
+
|
480
|
+
begin
|
481
|
+
registry.deploy( env_name.name, env_name.tag ) do |img|
|
482
|
+
new_name = NameFactory.new(env_name).identity
|
483
|
+
img.tag( repo: new_name.name, tag: new_name.tag )
|
484
|
+
end
|
485
|
+
rescue => msg
|
486
|
+
rbld_print.trace( msg )
|
487
|
+
raise EnvironmentDeploymentFailure, @cfg.remote!
|
488
|
+
end
|
489
|
+
|
490
|
+
@cache.refresh!
|
491
|
+
end
|
492
|
+
|
493
|
+
def publish(env_name)
|
494
|
+
env = unmodified_env( env_name )
|
495
|
+
|
496
|
+
rbld_print.progress "Checking for collisions..."
|
497
|
+
|
498
|
+
raise EnvironmentPublishCollision, env_name \
|
499
|
+
unless registry.search( env_name.name, env_name.tag ).empty?
|
500
|
+
|
501
|
+
begin
|
502
|
+
rbld_print.progress "Publishing on #{@cfg.remote!}..."
|
503
|
+
registry.publish( env.name, env.tag, env.img.api_obj )
|
504
|
+
rescue => msg
|
505
|
+
rbld_print.trace( msg )
|
506
|
+
raise EnvironmentPublishFailure, @cfg.remote!
|
507
|
+
end
|
508
|
+
end
|
509
|
+
|
510
|
+
def run(env_name, cmd)
|
511
|
+
env = existing_env( env_name )
|
512
|
+
run_env_disposable( env, cmd )
|
513
|
+
@cache.refresh!
|
514
|
+
@errno
|
515
|
+
end
|
516
|
+
|
517
|
+
def modify!(env_name, cmd)
|
518
|
+
env = existing_env( env_name )
|
519
|
+
|
520
|
+
rbld_print.progress_start 'Initializing environment'
|
521
|
+
|
522
|
+
if env.modified?
|
523
|
+
rbld_log.info("Running container #{env.modification_container}")
|
524
|
+
rerun_modification_cont(env, cmd)
|
525
|
+
else
|
526
|
+
rbld_log.info("Running environment #{env.img}")
|
527
|
+
rbld_print.progress_end
|
528
|
+
run_env(env, cmd)
|
529
|
+
end
|
530
|
+
@cache.refresh!
|
531
|
+
@errno
|
532
|
+
end
|
533
|
+
|
534
|
+
def commit!(env_name, new_tag)
|
535
|
+
env = existing_env( env_name )
|
536
|
+
|
537
|
+
new_name = Rebuild::Utils::FullImageName.new( env_name.name, new_tag )
|
538
|
+
nonexisting_env(new_name)
|
539
|
+
|
540
|
+
if env.modified?
|
541
|
+
rbld_log.info("Committing container #{env.modification_container}")
|
542
|
+
rbld_print.progress "Creating new environment #{new_name}..."
|
543
|
+
|
544
|
+
names = NameFactory.new( new_name )
|
545
|
+
env.modification_container.flatten( names.identity )
|
546
|
+
env.rerun_img.remove! if env.rerun_img
|
547
|
+
else
|
548
|
+
raise NoChangesToCommit, env_name.full
|
549
|
+
end
|
550
|
+
|
551
|
+
@cache.refresh!
|
552
|
+
end
|
553
|
+
|
554
|
+
def checkout!(env_name)
|
555
|
+
env = existing_env( env_name )
|
556
|
+
|
557
|
+
if env.modified?
|
558
|
+
rbld_log.info("Removing container #{env.modification_container}")
|
559
|
+
env.modification_container.remove!
|
560
|
+
end
|
561
|
+
|
562
|
+
env.rerun_img.remove! if env.rerun_img
|
563
|
+
@cache.refresh!
|
564
|
+
end
|
565
|
+
|
566
|
+
def create!(base, basefile, env_name)
|
567
|
+
begin
|
568
|
+
nonexisting_env(env_name)
|
569
|
+
|
570
|
+
rbld_print.progress "Building environment..."
|
571
|
+
|
572
|
+
context = basefile.nil? ? DockerContext.from_image(base) \
|
573
|
+
: DockerContext.from_file(basefile)
|
574
|
+
|
575
|
+
new_img = nil
|
576
|
+
|
577
|
+
context.prepare do |tar|
|
578
|
+
opts = { t: NameFactory.new(env_name).identity,
|
579
|
+
rm: true }
|
580
|
+
new_img = Docker::Image.build_from_tar( tar, opts ) do |v|
|
581
|
+
begin
|
582
|
+
if ( log = JSON.parse( v ) ) && log.has_key?( "stream" )
|
583
|
+
rbld_print.raw_trace( log["stream"] )
|
584
|
+
end
|
585
|
+
rescue
|
586
|
+
end
|
587
|
+
end
|
588
|
+
end
|
589
|
+
rescue Docker::Error::DockerError => msg
|
590
|
+
new_img.remove( :force => true ) if new_img
|
591
|
+
rbld_print.trace( msg )
|
592
|
+
raise EnvironmentCreateFailure, "#{env_name.full}"
|
593
|
+
end
|
594
|
+
|
595
|
+
@cache.refresh!
|
596
|
+
end
|
597
|
+
|
598
|
+
def_delegator :@cache, :all, :environments
|
599
|
+
|
600
|
+
private
|
601
|
+
|
602
|
+
def tweak_excon
|
603
|
+
# docker-api use Excon to issue HTTP requests
|
604
|
+
# and default Excon timeouts which are 60 seconds
|
605
|
+
# apply to all docker-api actions.
|
606
|
+
# Some long-running actions like image build may
|
607
|
+
# take more than 1 minute so timeout needs to be
|
608
|
+
# increased
|
609
|
+
Excon.defaults[:write_timeout] = 600
|
610
|
+
Excon.defaults[:read_timeout] = 600
|
611
|
+
end
|
612
|
+
|
613
|
+
def check_connectivity
|
614
|
+
begin
|
615
|
+
@docker_api.validate_version!
|
616
|
+
rescue Docker::Error::VersionError => msg
|
617
|
+
raise UnsupportedDockerService, msg
|
618
|
+
end
|
619
|
+
end
|
620
|
+
|
621
|
+
def registry
|
622
|
+
@registry ||= Rebuild::Registry::API.new( @cfg.remote! )
|
623
|
+
@registry
|
624
|
+
end
|
625
|
+
|
626
|
+
def run_external(cmdline)
|
627
|
+
rbld_log.info("Executing external command #{cmdline}")
|
628
|
+
system( cmdline )
|
629
|
+
@errno = $?.exitstatus
|
630
|
+
rbld_log.info( "External command returned with code #{@errno}" )
|
631
|
+
end
|
632
|
+
|
633
|
+
def run_settings(env, cmd, opts = {})
|
634
|
+
%Q{ -i #{STDIN.tty? ? '-t' : ''} \
|
635
|
+
-v #{Dir.home}:#{Dir.home} \
|
636
|
+
-e REBUILD_USER_ID=#{Process.uid} \
|
637
|
+
-e REBUILD_GROUP_ID=#{Process.gid} \
|
638
|
+
-e REBUILD_USER_NAME=#{Etc.getlogin} \
|
639
|
+
-e REBUILD_GROUP_NAME=#{Etc.getgrgid(Process.gid)[:name]} \
|
640
|
+
-e REBUILD_USER_HOME=#{Dir.home} \
|
641
|
+
-e REBUILD_PWD=#{Dir.pwd} \
|
642
|
+
--security-opt label:disable \
|
643
|
+
#{opts[:rerun] ? env.rerun_img.id : env.img.id} \
|
644
|
+
"#{cmd.join(' ')}" \
|
645
|
+
}
|
646
|
+
end
|
647
|
+
|
648
|
+
def run_env_disposable(env, cmd)
|
649
|
+
env.execution_container.remove! if env.execution_container
|
650
|
+
names = NameFactory.new(env)
|
651
|
+
|
652
|
+
cmdline = %Q{
|
653
|
+
docker run \
|
654
|
+
--rm \
|
655
|
+
--name #{names.running} \
|
656
|
+
--hostname #{names.hostname} \
|
657
|
+
#{run_settings( env, cmd )} \
|
658
|
+
}
|
659
|
+
|
660
|
+
run_external( cmdline )
|
661
|
+
end
|
662
|
+
|
663
|
+
def run_env(env, cmd, opts = {})
|
664
|
+
names = NameFactory.new(env)
|
665
|
+
|
666
|
+
cmdline = %Q{
|
667
|
+
docker run \
|
668
|
+
--name #{names.modified} \
|
669
|
+
--hostname #{names.modified_hostname} \
|
670
|
+
#{run_settings( env, cmd, opts )} \
|
671
|
+
}
|
672
|
+
|
673
|
+
run_external( cmdline )
|
674
|
+
end
|
675
|
+
|
676
|
+
def rerun_modification_cont(env, cmd)
|
677
|
+
rbld_print.progress_tick
|
678
|
+
|
679
|
+
names = NameFactory.new( env )
|
680
|
+
new_img = env.modification_container.commit( names.rerun )
|
681
|
+
|
682
|
+
rbld_print.progress_tick
|
683
|
+
|
684
|
+
#Remove old re-run image in case it became dangling
|
685
|
+
@cache.dangling.each(&:remove)
|
686
|
+
|
687
|
+
rbld_print.progress_end
|
688
|
+
|
689
|
+
@cache.refresh!
|
690
|
+
|
691
|
+
run_env( @cache.get(env), cmd, rerun: true )
|
692
|
+
end
|
693
|
+
|
694
|
+
def existing_env(name)
|
695
|
+
env = @cache.get(name)
|
696
|
+
raise EnvironmentNotKnown, name.full unless env
|
697
|
+
env
|
698
|
+
end
|
699
|
+
|
700
|
+
def unmodified_env(name)
|
701
|
+
env = existing_env( name )
|
702
|
+
raise EnvironmentIsModified if env.modified?
|
703
|
+
env
|
704
|
+
end
|
705
|
+
|
706
|
+
def nonexisting_env(name)
|
707
|
+
raise EnvironmentAlreadyExists, name.full if @cache.get(name)
|
708
|
+
end
|
709
|
+
end
|
710
|
+
end
|