rbld 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/cli/bin/rbld +35 -0
- data/cli/lib/bootstrap/re-build-bootstrap-utils +30 -0
- data/cli/lib/bootstrap/re-build-entry-point +75 -0
- data/cli/lib/bootstrap/re-build-env-prepare +52 -0
- data/cli/lib/bootstrap/rebuild.rc +6 -0
- data/cli/lib/commands/rbld_checkout.rb +14 -0
- data/cli/lib/commands/rbld_commit.rb +36 -0
- data/cli/lib/commands/rbld_create.rb +50 -0
- data/cli/lib/commands/rbld_deploy.rb +16 -0
- data/cli/lib/commands/rbld_list.rb +12 -0
- data/cli/lib/commands/rbld_load.rb +25 -0
- data/cli/lib/commands/rbld_modify.rb +22 -0
- data/cli/lib/commands/rbld_publish.rb +15 -0
- data/cli/lib/commands/rbld_rm.rb +14 -0
- data/cli/lib/commands/rbld_run.rb +24 -0
- data/cli/lib/commands/rbld_save.rb +28 -0
- data/cli/lib/commands/rbld_search.rb +13 -0
- data/cli/lib/commands/rbld_status.rb +12 -0
- data/cli/lib/rbld_commands.rb +207 -0
- data/cli/lib/rbld_config.rb +20 -0
- data/cli/lib/rbld_engine.rb +710 -0
- data/cli/lib/rbld_log.rb +26 -0
- data/cli/lib/rbld_print.rb +66 -0
- data/cli/lib/rbld_registry.rb +100 -0
- data/cli/lib/rbld_utils.rb +54 -0
- data/tools/rebuild-conf/Rakefile +19 -0
- metadata +294 -0
@@ -0,0 +1,28 @@
|
|
1
|
+
module Rebuild::CLI
|
2
|
+
class RbldSaveCommand < Command
|
3
|
+
private
|
4
|
+
|
5
|
+
def default_file(env)
|
6
|
+
"#{env.name}-#{env.tag}.rbld"
|
7
|
+
end
|
8
|
+
|
9
|
+
public
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
@usage = "save [OPTIONS] [ENVIRONMENT] [FILE]"
|
13
|
+
@description = "Save local environment to file"
|
14
|
+
end
|
15
|
+
|
16
|
+
def run(parameters)
|
17
|
+
env, file = parameters
|
18
|
+
env = Environment.new( env )
|
19
|
+
file = default_file( env ) if !file or file.empty?
|
20
|
+
rbld_log.info("Going to save #{env} to #{file}")
|
21
|
+
|
22
|
+
warn_if_modified( env, 'saving' )
|
23
|
+
engine_api.save(env, file)
|
24
|
+
|
25
|
+
rbld_print.progress "Successfully saved environment #{env} to #{file}\n"
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module Rebuild::CLI
|
2
|
+
class RbldSearchCommand < Command
|
3
|
+
def initialize
|
4
|
+
@usage = "search [OPTIONS] [NAME[:TAG]|PREFIX]"
|
5
|
+
@description = "Search remote registry for published environments"
|
6
|
+
end
|
7
|
+
|
8
|
+
def run(parameters)
|
9
|
+
env = Environment.new(parameters[0], allow_empty: true)
|
10
|
+
print_names( engine_api.search( env ) )
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
module Rebuild::CLI
|
2
|
+
class RbldStatusCommand < Command
|
3
|
+
def initialize
|
4
|
+
@usage = "status [OPTIONS]"
|
5
|
+
@description = "List modified environments"
|
6
|
+
end
|
7
|
+
|
8
|
+
def run(parameters)
|
9
|
+
print_names( engine_api.environments.select( &:modified? ), 'modified: ' )
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,207 @@
|
|
1
|
+
require_relative 'rbld_log'
|
2
|
+
require_relative 'rbld_utils'
|
3
|
+
require_relative 'rbld_engine'
|
4
|
+
|
5
|
+
module Rebuild::CLI
|
6
|
+
extend Rebuild::Utils::Errors
|
7
|
+
|
8
|
+
rebuild_errors \
|
9
|
+
EnvironmentNameEmpty: 'Environment name not specified',
|
10
|
+
EnvironmentNameWithoutTagExpected: 'Environment tag must not be specified',
|
11
|
+
EnvironmentNameError: 'Invalid %s, it may contain a-z, A-Z, 0-9, - and _ characters only',
|
12
|
+
HandlerClassNameError: '%s'
|
13
|
+
|
14
|
+
class Environment
|
15
|
+
def initialize(env, opts = {})
|
16
|
+
if env.respond_to?( :name ) && env.respond_to?( :tag )
|
17
|
+
@name, @tag = env.name, env.tag
|
18
|
+
else
|
19
|
+
deduce_name_tag( env, opts )
|
20
|
+
validate_name_tag(opts)
|
21
|
+
end
|
22
|
+
@full = "#{@name}:#{@tag}"
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.validate_component( name, value )
|
26
|
+
raise EnvironmentNameError, "#{name} (#{value})" \
|
27
|
+
unless value.match( /^[[:alnum:]\_\-]*$/ )
|
28
|
+
end
|
29
|
+
|
30
|
+
def to_s
|
31
|
+
@full
|
32
|
+
end
|
33
|
+
|
34
|
+
attr_reader :name, :tag, :full
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def parse_name_tag(cli_param, opts)
|
39
|
+
if opts[:allow_empty] && (!cli_param || cli_param.empty?)
|
40
|
+
@name, @tag = '', ''
|
41
|
+
elsif !cli_param || cli_param.empty?
|
42
|
+
raise EnvironmentNameEmpty
|
43
|
+
else
|
44
|
+
@name, @tag = cli_param.match( /^([^:]*):?(.*)/ ).captures
|
45
|
+
@tag = '' if @name.empty?
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def deduce_name_tag(cli_param, opts)
|
50
|
+
parse_name_tag( cli_param, opts )
|
51
|
+
|
52
|
+
raise EnvironmentNameWithoutTagExpected \
|
53
|
+
if opts[:force_no_tag] && !@tag.empty?
|
54
|
+
|
55
|
+
@tag = 'initial' if @tag.empty? && !opts[:allow_empty]
|
56
|
+
end
|
57
|
+
|
58
|
+
def validate_name_tag(opts)
|
59
|
+
raise EnvironmentNameEmpty if @name.empty? && !opts[:allow_empty]
|
60
|
+
self.class.validate_component( "environment name", @name ) unless @name.empty?
|
61
|
+
self.class.validate_component( "environment tag", @tag ) unless @tag.empty?
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
class Commands
|
66
|
+
extend Enumerable
|
67
|
+
|
68
|
+
private
|
69
|
+
|
70
|
+
def self.deduce_cmd_name(handler_class)
|
71
|
+
match = handler_class.name.match(/Rbld(.*)Command/)
|
72
|
+
return nil unless match
|
73
|
+
match.captures[0].downcase
|
74
|
+
end
|
75
|
+
|
76
|
+
def self.handler!(command)
|
77
|
+
@handler_classes.each do |klass|
|
78
|
+
return klass.new if command == deduce_cmd_name( klass )
|
79
|
+
end
|
80
|
+
|
81
|
+
raise "Unknown command: #{command}"
|
82
|
+
end
|
83
|
+
|
84
|
+
@handler_classes = []
|
85
|
+
|
86
|
+
public
|
87
|
+
|
88
|
+
def self.register_handler_class(klass)
|
89
|
+
unless deduce_cmd_name( klass )
|
90
|
+
raise HandlerClassNameError, klass.name
|
91
|
+
end
|
92
|
+
|
93
|
+
@handler_classes << klass
|
94
|
+
end
|
95
|
+
|
96
|
+
def self.each
|
97
|
+
@handler_classes.each { |klass| yield( deduce_cmd_name( klass ) ) }
|
98
|
+
end
|
99
|
+
|
100
|
+
def self.usage(command)
|
101
|
+
handler!( command ).usage
|
102
|
+
end
|
103
|
+
|
104
|
+
def self.run(command, parameters)
|
105
|
+
handler = handler!( command )
|
106
|
+
handler.run( parameters )
|
107
|
+
handler.errno
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
class Command
|
112
|
+
|
113
|
+
private
|
114
|
+
|
115
|
+
def self.inherited( handler_class )
|
116
|
+
Commands.register_handler_class( handler_class )
|
117
|
+
rbld_log.info( "Command handler class #{handler_class} registered" )
|
118
|
+
end
|
119
|
+
|
120
|
+
def options_text
|
121
|
+
options = (@options || []) + [["-h, --help", "Print usage"]]
|
122
|
+
text = ""
|
123
|
+
options.each { |o| text << " #{o[0].ljust(30)}#{o[1]}\n" }
|
124
|
+
text
|
125
|
+
end
|
126
|
+
|
127
|
+
def replace_argv(parameters)
|
128
|
+
orig_argv = ARGV.clone
|
129
|
+
ARGV.clear
|
130
|
+
parameters.each { |x| ARGV << x }
|
131
|
+
yield
|
132
|
+
ARGV.clear
|
133
|
+
orig_argv.each { |x| ARGV << x }
|
134
|
+
end
|
135
|
+
|
136
|
+
def print_names(names, prefix = '')
|
137
|
+
strings = names.map { |n| Environment.new(n).full }
|
138
|
+
puts
|
139
|
+
strings.sort.each { |s| puts " #{prefix}#{s}"}
|
140
|
+
puts
|
141
|
+
end
|
142
|
+
|
143
|
+
def engine_api
|
144
|
+
@engine_api ||= Rebuild::Engine::API.new
|
145
|
+
@engine_api
|
146
|
+
end
|
147
|
+
|
148
|
+
def warn_if_modified(env, action)
|
149
|
+
rbld_print.warning "Environment is modified, #{action} original version" \
|
150
|
+
if engine_api.environments.select( &:modified? ).include?( env )
|
151
|
+
end
|
152
|
+
|
153
|
+
def get_cmdline_tail(parameters)
|
154
|
+
parameters.shift if parameters[0] == '--'
|
155
|
+
parameters
|
156
|
+
end
|
157
|
+
|
158
|
+
def format_usage_text
|
159
|
+
text = ""
|
160
|
+
if @usage.respond_to?(:each)
|
161
|
+
text << "\n"
|
162
|
+
@usage.each do |mode|
|
163
|
+
text << "\n rbld #{mode[:syntax]}\n\n" \
|
164
|
+
" #{mode[:description]}\n"
|
165
|
+
end
|
166
|
+
else
|
167
|
+
text << "rbld #{@usage}\n"
|
168
|
+
end
|
169
|
+
text
|
170
|
+
end
|
171
|
+
|
172
|
+
public
|
173
|
+
|
174
|
+
attr_reader :errno
|
175
|
+
|
176
|
+
def usage
|
177
|
+
puts <<END_USAGE
|
178
|
+
|
179
|
+
Usage: #{format_usage_text}
|
180
|
+
#{@description}
|
181
|
+
|
182
|
+
#{options_text}
|
183
|
+
END_USAGE
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
class Main
|
188
|
+
|
189
|
+
def self.usage
|
190
|
+
usage_text = <<USAGE
|
191
|
+
Usage:
|
192
|
+
rbld help Show this help screen
|
193
|
+
rbld help COMMAND Show help for COMMAND
|
194
|
+
rbld COMMAND [PARAMS] Run COMMAND with PARAMS
|
195
|
+
|
196
|
+
rebuild: Zero-dependency, reproducible build environments
|
197
|
+
|
198
|
+
Commands:
|
199
|
+
|
200
|
+
USAGE
|
201
|
+
|
202
|
+
Commands.sort.each { |cmd| usage_text << " #{cmd}\n"}
|
203
|
+
|
204
|
+
usage_text
|
205
|
+
end
|
206
|
+
end
|
207
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'parseconfig'
|
2
|
+
|
3
|
+
module Rebuild
|
4
|
+
class Config
|
5
|
+
def initialize()
|
6
|
+
cfg_file = File.join( Dir.home, '.rbld', 'rebuild.conf' )
|
7
|
+
|
8
|
+
if File.exist?( cfg_file )
|
9
|
+
cfg = ParseConfig.new( cfg_file )
|
10
|
+
rname = cfg['REMOTE_NAME']
|
11
|
+
@remote = rname ? cfg["REMOTE_#{rname}"] : nil
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def remote!
|
16
|
+
raise 'Remote not defined' unless @remote
|
17
|
+
@remote
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,710 @@
|
|
1
|
+
require 'docker'
|
2
|
+
require 'etc'
|
3
|
+
require 'thread'
|
4
|
+
require 'forwardable'
|
5
|
+
require_relative 'rbld_log'
|
6
|
+
require_relative 'rbld_config'
|
7
|
+
require_relative 'rbld_utils'
|
8
|
+
require_relative 'rbld_print'
|
9
|
+
require_relative 'rbld_registry'
|
10
|
+
|
11
|
+
module Rebuild::Engine
|
12
|
+
extend Rebuild::Utils::Errors
|
13
|
+
|
14
|
+
class NamedDockerImage
|
15
|
+
extend Forwardable
|
16
|
+
|
17
|
+
def_delegators :@api_obj, :id
|
18
|
+
attr_reader :api_obj
|
19
|
+
|
20
|
+
def initialize(name, api_obj)
|
21
|
+
@name, @api_obj = name, api_obj
|
22
|
+
end
|
23
|
+
|
24
|
+
def remove!
|
25
|
+
@api_obj.remove( name: @name.to_s )
|
26
|
+
end
|
27
|
+
|
28
|
+
def tag
|
29
|
+
@api_obj.tag( repo: @name.repo,
|
30
|
+
tag: @name.tag )
|
31
|
+
end
|
32
|
+
|
33
|
+
def identity
|
34
|
+
@name.to_s
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
class NamedDockerContainer
|
39
|
+
def initialize(name, api_obj)
|
40
|
+
@name, @api_obj = name, api_obj
|
41
|
+
end
|
42
|
+
|
43
|
+
def remove!
|
44
|
+
@api_obj.delete( force: true )
|
45
|
+
end
|
46
|
+
|
47
|
+
def flatten(img_name)
|
48
|
+
data_queue = Queue.new
|
49
|
+
new_img = nil
|
50
|
+
|
51
|
+
exporter = Thread.new do
|
52
|
+
@api_obj.export { |chunk| data_queue << chunk }
|
53
|
+
data_queue << ''
|
54
|
+
end
|
55
|
+
|
56
|
+
importer = Thread.new do
|
57
|
+
opts = commit_opts(img_name)
|
58
|
+
new_img = Docker::Image.import_stream(opts) { data_queue.pop }
|
59
|
+
end
|
60
|
+
|
61
|
+
exporter.join
|
62
|
+
importer.join
|
63
|
+
|
64
|
+
rbld_log.info("Created image #{new_img} from #{@api_obj}")
|
65
|
+
remove!
|
66
|
+
|
67
|
+
new_img
|
68
|
+
end
|
69
|
+
|
70
|
+
def commit(img_name)
|
71
|
+
if squash_needed?
|
72
|
+
flatten( img_name )
|
73
|
+
else
|
74
|
+
new_img = @api_obj.commit( commit_opts( img_name ) )
|
75
|
+
rbld_log.info("Created image #{new_img} from #{@api_obj}")
|
76
|
+
remove!
|
77
|
+
new_img
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
private
|
82
|
+
|
83
|
+
def squash_needed?
|
84
|
+
#Different FS backends have different limitations for
|
85
|
+
#maximum number of layers, 40 looks like small enough
|
86
|
+
#to be supported by all possible configurations
|
87
|
+
Docker::Image.get(@api_obj.info["ImageID"]).history.size >= 40
|
88
|
+
end
|
89
|
+
|
90
|
+
def commit_opts(img_name)
|
91
|
+
{repo: img_name.repo,
|
92
|
+
tag: img_name.tag,
|
93
|
+
changes: ["LABEL re-build-environment=true",
|
94
|
+
"ENTRYPOINT [\"/rebuild/re-build-entry-point\"]"]}
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
class NameFactory
|
99
|
+
def initialize(env)
|
100
|
+
@env = env
|
101
|
+
end
|
102
|
+
|
103
|
+
def identity
|
104
|
+
Rebuild::Utils::FullImageName.new("rbe-#{@env.name}", @env.tag)
|
105
|
+
end
|
106
|
+
|
107
|
+
def rerun
|
108
|
+
Rebuild::Utils::FullImageName.new("rbr-#{@env.name}-rt-#{@env.tag}", 'initial')
|
109
|
+
end
|
110
|
+
|
111
|
+
def running
|
112
|
+
container_name(:running)
|
113
|
+
end
|
114
|
+
|
115
|
+
def modified
|
116
|
+
container_name(:dirty)
|
117
|
+
end
|
118
|
+
|
119
|
+
def hostname
|
120
|
+
"#{@env.name}-#{@env.tag}"
|
121
|
+
end
|
122
|
+
|
123
|
+
def modified_hostname
|
124
|
+
"#{hostname}-M"
|
125
|
+
end
|
126
|
+
|
127
|
+
private
|
128
|
+
|
129
|
+
def container_name(type)
|
130
|
+
"rbe-#{type.to_s.chars.first}-#{@env.name}-rt-#{@env.tag}"
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
class Environment
|
135
|
+
def self.from_image(img_name, api_obj)
|
136
|
+
if match = img_name.match(/^rbe-(.*):(.*)/)
|
137
|
+
new( *match.captures, NamedDockerImage.new( img_name, api_obj ) )
|
138
|
+
else
|
139
|
+
nil
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
def attach_container(cont_name, api_obj)
|
144
|
+
try_attach_container(:dirty, cont_name, api_obj) ||
|
145
|
+
try_attach_container(:running, cont_name, api_obj)
|
146
|
+
end
|
147
|
+
|
148
|
+
def attach_rerun_image(img_name, api_obj)
|
149
|
+
match = img_name.match(/^rbr-(.*)-rt-(.*):initial/)
|
150
|
+
if my_object?( match )
|
151
|
+
@rerun_img = NamedDockerImage.new( img_name, api_obj )
|
152
|
+
true
|
153
|
+
else
|
154
|
+
false
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
def modified?
|
159
|
+
!@cont[:dirty].nil?
|
160
|
+
end
|
161
|
+
|
162
|
+
def execution_container
|
163
|
+
@cont[:running]
|
164
|
+
end
|
165
|
+
|
166
|
+
def modification_container
|
167
|
+
@cont[:dirty]
|
168
|
+
end
|
169
|
+
|
170
|
+
def ==(other)
|
171
|
+
(other.name == @name) && (other.tag == @tag)
|
172
|
+
end
|
173
|
+
|
174
|
+
attr_reader :name, :tag, :img, :rerun_img
|
175
|
+
|
176
|
+
private
|
177
|
+
|
178
|
+
def initialize(name, tag, img)
|
179
|
+
@name, @tag, @img, @cont = name, tag, img, {}
|
180
|
+
end
|
181
|
+
|
182
|
+
def my_object?(match)
|
183
|
+
match && (match[1] == @name) && (match[2] == @tag)
|
184
|
+
end
|
185
|
+
|
186
|
+
def try_attach_container(type, cont_name, api_obj)
|
187
|
+
match = cont_name.match(/^\/rbe-#{type.to_s.chars.first}-(.*)-rt-(.*)/)
|
188
|
+
if my_object?( match )
|
189
|
+
@cont[type] = NamedDockerContainer.new( cont_name, api_obj )
|
190
|
+
true
|
191
|
+
else
|
192
|
+
false
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
private_class_method :new
|
197
|
+
end
|
198
|
+
|
199
|
+
class PresentEnvironments
|
200
|
+
include Enumerable
|
201
|
+
extend Forwardable
|
202
|
+
|
203
|
+
def initialize(api_module = Docker)
|
204
|
+
@api_module = api_module
|
205
|
+
refresh!
|
206
|
+
end
|
207
|
+
|
208
|
+
def refresh!
|
209
|
+
cache_images
|
210
|
+
attach_containers
|
211
|
+
attach_rerun_images
|
212
|
+
end
|
213
|
+
|
214
|
+
def_delegators :@all, :each
|
215
|
+
attr_reader :all
|
216
|
+
|
217
|
+
def dangling
|
218
|
+
rbld_images(dangling: ['true'])
|
219
|
+
end
|
220
|
+
|
221
|
+
def get(name)
|
222
|
+
find { |e| e == name }
|
223
|
+
end
|
224
|
+
|
225
|
+
private
|
226
|
+
|
227
|
+
def rbld_obj_filter
|
228
|
+
{ label: ["re-build-environment=true"] }
|
229
|
+
end
|
230
|
+
|
231
|
+
def rbld_images(filters = nil)
|
232
|
+
filters = rbld_obj_filter.merge( filters || {} )
|
233
|
+
@api_module::Image.all( :filters => filters.to_json )
|
234
|
+
end
|
235
|
+
|
236
|
+
def rbld_containers
|
237
|
+
@api_module::Container.all( all: true, filters: rbld_obj_filter.to_json )
|
238
|
+
end
|
239
|
+
|
240
|
+
def cache_images
|
241
|
+
@all = []
|
242
|
+
rbld_images.each do |img|
|
243
|
+
rbld_log.debug("Found docker image #{img}")
|
244
|
+
img.info['RepoTags'].each { |tag| @all << Environment.from_image( tag, img ) }
|
245
|
+
end
|
246
|
+
@all.compact!
|
247
|
+
end
|
248
|
+
|
249
|
+
def attach_containers
|
250
|
+
rbld_containers.each do |cont|
|
251
|
+
rbld_log.debug("Found docker container #{cont}")
|
252
|
+
cont.info['Names'].each do |name|
|
253
|
+
@all.find { |e| e.attach_container( name, cont ) }
|
254
|
+
end
|
255
|
+
end
|
256
|
+
end
|
257
|
+
|
258
|
+
def attach_rerun_images
|
259
|
+
rbld_images.each do |img|
|
260
|
+
rbld_log.debug("Found docker image #{img}")
|
261
|
+
img.info['RepoTags'].each do |tag|
|
262
|
+
@all.find { |e| e.attach_rerun_image( tag, img ) }
|
263
|
+
end
|
264
|
+
end
|
265
|
+
end
|
266
|
+
end
|
267
|
+
|
268
|
+
rebuild_errors \
|
269
|
+
UnsupportedDockerService: 'Unsupported docker service: %s',
|
270
|
+
EnvironmentIsModified: 'Environment is modified, commit or checkout first',
|
271
|
+
EnvironmentNotKnown: 'Unknown environment %s',
|
272
|
+
NoChangesToCommit: 'No changes to commit for %s',
|
273
|
+
EnvironmentLoadFailure: 'Failed to load environment from %s',
|
274
|
+
EnvironmentSaveFailure: 'Failed to save environment %s to %s',
|
275
|
+
EnvironmentDeploymentFailure: 'Failed to deploy from %s',
|
276
|
+
EnvironmentAlreadyExists: 'Environment %s already exists',
|
277
|
+
EnvironmentNotFoundInTheRegistry: 'Environment %s does not exist in the registry',
|
278
|
+
RegistrySearchFailed: 'Failed to search in %s',
|
279
|
+
EnvironmentPublishCollision: 'Environment %s already published',
|
280
|
+
EnvironmentPublishFailure: 'Failed to publish on %s',
|
281
|
+
EnvironmentCreateFailure: 'Failed to create %s'
|
282
|
+
|
283
|
+
class EnvironmentFile
|
284
|
+
def initialize(filename, docker_api = Docker)
|
285
|
+
@filename, @docker_api = filename, docker_api
|
286
|
+
end
|
287
|
+
|
288
|
+
def load!
|
289
|
+
begin
|
290
|
+
with_gzip_reader { |gz| Docker::Image.load(gz) }
|
291
|
+
rescue => msg
|
292
|
+
rbld_print.trace( msg )
|
293
|
+
raise EnvironmentLoadFailure, @filename
|
294
|
+
end
|
295
|
+
end
|
296
|
+
|
297
|
+
def save!(name, identity)
|
298
|
+
begin
|
299
|
+
with_gzip_writer do |gz|
|
300
|
+
Docker::Image.save_stream( identity ) { |chunk| gz.write chunk }
|
301
|
+
end
|
302
|
+
rescue => msg
|
303
|
+
rbld_print.trace( msg )
|
304
|
+
raise EnvironmentSaveFailure, [name, @filename]
|
305
|
+
end
|
306
|
+
end
|
307
|
+
|
308
|
+
private
|
309
|
+
|
310
|
+
def with_gzip_writer
|
311
|
+
begin
|
312
|
+
File.open(@filename, 'w') do |f|
|
313
|
+
gz = Zlib::GzipWriter.new(f)
|
314
|
+
begin
|
315
|
+
yield gz
|
316
|
+
ensure
|
317
|
+
gz.close
|
318
|
+
end
|
319
|
+
end
|
320
|
+
rescue
|
321
|
+
FileUtils::safe_unlink( @filename )
|
322
|
+
raise
|
323
|
+
end
|
324
|
+
end
|
325
|
+
|
326
|
+
def with_gzip_reader
|
327
|
+
Zlib::GzipReader.open( @filename ) do |gz|
|
328
|
+
begin
|
329
|
+
yield gz
|
330
|
+
ensure
|
331
|
+
gz.close
|
332
|
+
end
|
333
|
+
end
|
334
|
+
end
|
335
|
+
end
|
336
|
+
|
337
|
+
class DockerContext
|
338
|
+
def self.from_file(file)
|
339
|
+
base = %Q{
|
340
|
+
FROM scratch
|
341
|
+
ADD #{file} /
|
342
|
+
}
|
343
|
+
|
344
|
+
new( base, file )
|
345
|
+
end
|
346
|
+
|
347
|
+
def self.from_image(img)
|
348
|
+
base = %Q{
|
349
|
+
FROM #{img}
|
350
|
+
}
|
351
|
+
|
352
|
+
new( base )
|
353
|
+
end
|
354
|
+
|
355
|
+
def initialize(base, basefile = nil)
|
356
|
+
@base, @basefile = base, basefile
|
357
|
+
end
|
358
|
+
|
359
|
+
def prepare
|
360
|
+
tarfile_name = Dir::Tmpname.create('rbldctx') {}
|
361
|
+
|
362
|
+
rbld_log.info("Storing context in #{tarfile_name}")
|
363
|
+
|
364
|
+
File.open(tarfile_name, 'wb+') do |tarfile|
|
365
|
+
Gem::Package::TarWriter.new( tarfile ) do |tar|
|
366
|
+
|
367
|
+
files = bootstrap_file_pathnames
|
368
|
+
files << @basefile unless @basefile.nil?
|
369
|
+
|
370
|
+
files.each do |file_name|
|
371
|
+
tar.add_file(File.basename(file_name), 0640) do |t|
|
372
|
+
IO.copy_stream(file_name, t)
|
373
|
+
end
|
374
|
+
end
|
375
|
+
|
376
|
+
tar.add_file('Dockerfile', 0640) do |t|
|
377
|
+
t.write( dockerfile )
|
378
|
+
end
|
379
|
+
|
380
|
+
end
|
381
|
+
end
|
382
|
+
|
383
|
+
File.open(tarfile_name, 'r') { |f| yield f }
|
384
|
+
ensure
|
385
|
+
FileUtils::rm_f( tarfile_name )
|
386
|
+
end
|
387
|
+
|
388
|
+
private
|
389
|
+
|
390
|
+
def bootstrap_files
|
391
|
+
["re-build-bootstrap-utils",
|
392
|
+
"re-build-entry-point",
|
393
|
+
"re-build-env-prepare",
|
394
|
+
"rebuild.rc"]
|
395
|
+
end
|
396
|
+
|
397
|
+
def bootstrap_file_pathnames
|
398
|
+
src_path = File.join( File.dirname( __FILE__ ), "bootstrap" )
|
399
|
+
src_path = File.expand_path( src_path )
|
400
|
+
bootstrap_files.map { |f| File.join( src_path, f ) }
|
401
|
+
end
|
402
|
+
|
403
|
+
def dockerfile
|
404
|
+
# sync after chmod is needed because of an AuFS problem described in:
|
405
|
+
# https://github.com/docker/docker/issues/9547
|
406
|
+
%Q{
|
407
|
+
#{@base}
|
408
|
+
LABEL re-build-environment=true
|
409
|
+
COPY #{bootstrap_files.join(' ')} /rebuild/
|
410
|
+
RUN chown root:root \
|
411
|
+
/rebuild/re-build-env-prepare \
|
412
|
+
/rebuild/re-build-bootstrap-utils \
|
413
|
+
/rebuild/rebuild.rc && \
|
414
|
+
chmod 700 \
|
415
|
+
/rebuild/re-build-entry-point \
|
416
|
+
/rebuild/re-build-env-prepare && \
|
417
|
+
sync && \
|
418
|
+
chmod 644 \
|
419
|
+
/rebuild/rebuild.rc \
|
420
|
+
/rebuild/re-build-bootstrap-utils && \
|
421
|
+
sync && \
|
422
|
+
/rebuild/re-build-env-prepare
|
423
|
+
ENTRYPOINT ["/rebuild/re-build-entry-point"]
|
424
|
+
}.squeeze(' ')
|
425
|
+
end
|
426
|
+
|
427
|
+
private_class_method :new
|
428
|
+
end
|
429
|
+
|
430
|
+
class API
|
431
|
+
extend Forwardable
|
432
|
+
|
433
|
+
def initialize(docker_api = Docker, cfg = Rebuild::Config.new)
|
434
|
+
@docker_api, @cfg = docker_api, cfg
|
435
|
+
|
436
|
+
tweak_excon
|
437
|
+
check_connectivity
|
438
|
+
@cache = PresentEnvironments.new
|
439
|
+
end
|
440
|
+
|
441
|
+
def remove!(env_name)
|
442
|
+
env = unmodified_env( env_name )
|
443
|
+
env.img.remove!
|
444
|
+
@cache.refresh!
|
445
|
+
end
|
446
|
+
|
447
|
+
def load!(filename)
|
448
|
+
EnvironmentFile.new( filename ).load!
|
449
|
+
# If image with the same name but another
|
450
|
+
# ID existed before load it becomes dangling
|
451
|
+
# and should be ditched
|
452
|
+
@cache.dangling.each(&:remove)
|
453
|
+
@cache.refresh!
|
454
|
+
end
|
455
|
+
|
456
|
+
def save(env_name, filename)
|
457
|
+
env = existing_env( env_name )
|
458
|
+
EnvironmentFile.new( filename ).save!(env_name, env.img.identity)
|
459
|
+
end
|
460
|
+
|
461
|
+
def search(env_name)
|
462
|
+
rbld_print.progress "Searching in #{@cfg.remote!}..."
|
463
|
+
|
464
|
+
begin
|
465
|
+
registry.search( env_name.name, env_name.tag )
|
466
|
+
rescue => msg
|
467
|
+
rbld_print.trace( msg )
|
468
|
+
raise RegistrySearchFailed, @cfg.remote!
|
469
|
+
end
|
470
|
+
end
|
471
|
+
|
472
|
+
def deploy!(env_name)
|
473
|
+
nonexisting_env(env_name)
|
474
|
+
|
475
|
+
raise EnvironmentNotFoundInTheRegistry, env_name.full \
|
476
|
+
if registry.search( env_name.name, env_name.tag ).empty?
|
477
|
+
|
478
|
+
rbld_print.progress "Deploying from #{@cfg.remote!}..."
|
479
|
+
|
480
|
+
begin
|
481
|
+
registry.deploy( env_name.name, env_name.tag ) do |img|
|
482
|
+
new_name = NameFactory.new(env_name).identity
|
483
|
+
img.tag( repo: new_name.name, tag: new_name.tag )
|
484
|
+
end
|
485
|
+
rescue => msg
|
486
|
+
rbld_print.trace( msg )
|
487
|
+
raise EnvironmentDeploymentFailure, @cfg.remote!
|
488
|
+
end
|
489
|
+
|
490
|
+
@cache.refresh!
|
491
|
+
end
|
492
|
+
|
493
|
+
def publish(env_name)
|
494
|
+
env = unmodified_env( env_name )
|
495
|
+
|
496
|
+
rbld_print.progress "Checking for collisions..."
|
497
|
+
|
498
|
+
raise EnvironmentPublishCollision, env_name \
|
499
|
+
unless registry.search( env_name.name, env_name.tag ).empty?
|
500
|
+
|
501
|
+
begin
|
502
|
+
rbld_print.progress "Publishing on #{@cfg.remote!}..."
|
503
|
+
registry.publish( env.name, env.tag, env.img.api_obj )
|
504
|
+
rescue => msg
|
505
|
+
rbld_print.trace( msg )
|
506
|
+
raise EnvironmentPublishFailure, @cfg.remote!
|
507
|
+
end
|
508
|
+
end
|
509
|
+
|
510
|
+
def run(env_name, cmd)
|
511
|
+
env = existing_env( env_name )
|
512
|
+
run_env_disposable( env, cmd )
|
513
|
+
@cache.refresh!
|
514
|
+
@errno
|
515
|
+
end
|
516
|
+
|
517
|
+
def modify!(env_name, cmd)
|
518
|
+
env = existing_env( env_name )
|
519
|
+
|
520
|
+
rbld_print.progress_start 'Initializing environment'
|
521
|
+
|
522
|
+
if env.modified?
|
523
|
+
rbld_log.info("Running container #{env.modification_container}")
|
524
|
+
rerun_modification_cont(env, cmd)
|
525
|
+
else
|
526
|
+
rbld_log.info("Running environment #{env.img}")
|
527
|
+
rbld_print.progress_end
|
528
|
+
run_env(env, cmd)
|
529
|
+
end
|
530
|
+
@cache.refresh!
|
531
|
+
@errno
|
532
|
+
end
|
533
|
+
|
534
|
+
def commit!(env_name, new_tag)
|
535
|
+
env = existing_env( env_name )
|
536
|
+
|
537
|
+
new_name = Rebuild::Utils::FullImageName.new( env_name.name, new_tag )
|
538
|
+
nonexisting_env(new_name)
|
539
|
+
|
540
|
+
if env.modified?
|
541
|
+
rbld_log.info("Committing container #{env.modification_container}")
|
542
|
+
rbld_print.progress "Creating new environment #{new_name}..."
|
543
|
+
|
544
|
+
names = NameFactory.new( new_name )
|
545
|
+
env.modification_container.flatten( names.identity )
|
546
|
+
env.rerun_img.remove! if env.rerun_img
|
547
|
+
else
|
548
|
+
raise NoChangesToCommit, env_name.full
|
549
|
+
end
|
550
|
+
|
551
|
+
@cache.refresh!
|
552
|
+
end
|
553
|
+
|
554
|
+
def checkout!(env_name)
|
555
|
+
env = existing_env( env_name )
|
556
|
+
|
557
|
+
if env.modified?
|
558
|
+
rbld_log.info("Removing container #{env.modification_container}")
|
559
|
+
env.modification_container.remove!
|
560
|
+
end
|
561
|
+
|
562
|
+
env.rerun_img.remove! if env.rerun_img
|
563
|
+
@cache.refresh!
|
564
|
+
end
|
565
|
+
|
566
|
+
def create!(base, basefile, env_name)
|
567
|
+
begin
|
568
|
+
nonexisting_env(env_name)
|
569
|
+
|
570
|
+
rbld_print.progress "Building environment..."
|
571
|
+
|
572
|
+
context = basefile.nil? ? DockerContext.from_image(base) \
|
573
|
+
: DockerContext.from_file(basefile)
|
574
|
+
|
575
|
+
new_img = nil
|
576
|
+
|
577
|
+
context.prepare do |tar|
|
578
|
+
opts = { t: NameFactory.new(env_name).identity,
|
579
|
+
rm: true }
|
580
|
+
new_img = Docker::Image.build_from_tar( tar, opts ) do |v|
|
581
|
+
begin
|
582
|
+
if ( log = JSON.parse( v ) ) && log.has_key?( "stream" )
|
583
|
+
rbld_print.raw_trace( log["stream"] )
|
584
|
+
end
|
585
|
+
rescue
|
586
|
+
end
|
587
|
+
end
|
588
|
+
end
|
589
|
+
rescue Docker::Error::DockerError => msg
|
590
|
+
new_img.remove( :force => true ) if new_img
|
591
|
+
rbld_print.trace( msg )
|
592
|
+
raise EnvironmentCreateFailure, "#{env_name.full}"
|
593
|
+
end
|
594
|
+
|
595
|
+
@cache.refresh!
|
596
|
+
end
|
597
|
+
|
598
|
+
def_delegator :@cache, :all, :environments
|
599
|
+
|
600
|
+
private
|
601
|
+
|
602
|
+
def tweak_excon
|
603
|
+
# docker-api use Excon to issue HTTP requests
|
604
|
+
# and default Excon timeouts which are 60 seconds
|
605
|
+
# apply to all docker-api actions.
|
606
|
+
# Some long-running actions like image build may
|
607
|
+
# take more than 1 minute so timeout needs to be
|
608
|
+
# increased
|
609
|
+
Excon.defaults[:write_timeout] = 600
|
610
|
+
Excon.defaults[:read_timeout] = 600
|
611
|
+
end
|
612
|
+
|
613
|
+
def check_connectivity
|
614
|
+
begin
|
615
|
+
@docker_api.validate_version!
|
616
|
+
rescue Docker::Error::VersionError => msg
|
617
|
+
raise UnsupportedDockerService, msg
|
618
|
+
end
|
619
|
+
end
|
620
|
+
|
621
|
+
def registry
|
622
|
+
@registry ||= Rebuild::Registry::API.new( @cfg.remote! )
|
623
|
+
@registry
|
624
|
+
end
|
625
|
+
|
626
|
+
def run_external(cmdline)
|
627
|
+
rbld_log.info("Executing external command #{cmdline}")
|
628
|
+
system( cmdline )
|
629
|
+
@errno = $?.exitstatus
|
630
|
+
rbld_log.info( "External command returned with code #{@errno}" )
|
631
|
+
end
|
632
|
+
|
633
|
+
def run_settings(env, cmd, opts = {})
|
634
|
+
%Q{ -i #{STDIN.tty? ? '-t' : ''} \
|
635
|
+
-v #{Dir.home}:#{Dir.home} \
|
636
|
+
-e REBUILD_USER_ID=#{Process.uid} \
|
637
|
+
-e REBUILD_GROUP_ID=#{Process.gid} \
|
638
|
+
-e REBUILD_USER_NAME=#{Etc.getlogin} \
|
639
|
+
-e REBUILD_GROUP_NAME=#{Etc.getgrgid(Process.gid)[:name]} \
|
640
|
+
-e REBUILD_USER_HOME=#{Dir.home} \
|
641
|
+
-e REBUILD_PWD=#{Dir.pwd} \
|
642
|
+
--security-opt label:disable \
|
643
|
+
#{opts[:rerun] ? env.rerun_img.id : env.img.id} \
|
644
|
+
"#{cmd.join(' ')}" \
|
645
|
+
}
|
646
|
+
end
|
647
|
+
|
648
|
+
def run_env_disposable(env, cmd)
|
649
|
+
env.execution_container.remove! if env.execution_container
|
650
|
+
names = NameFactory.new(env)
|
651
|
+
|
652
|
+
cmdline = %Q{
|
653
|
+
docker run \
|
654
|
+
--rm \
|
655
|
+
--name #{names.running} \
|
656
|
+
--hostname #{names.hostname} \
|
657
|
+
#{run_settings( env, cmd )} \
|
658
|
+
}
|
659
|
+
|
660
|
+
run_external( cmdline )
|
661
|
+
end
|
662
|
+
|
663
|
+
def run_env(env, cmd, opts = {})
|
664
|
+
names = NameFactory.new(env)
|
665
|
+
|
666
|
+
cmdline = %Q{
|
667
|
+
docker run \
|
668
|
+
--name #{names.modified} \
|
669
|
+
--hostname #{names.modified_hostname} \
|
670
|
+
#{run_settings( env, cmd, opts )} \
|
671
|
+
}
|
672
|
+
|
673
|
+
run_external( cmdline )
|
674
|
+
end
|
675
|
+
|
676
|
+
def rerun_modification_cont(env, cmd)
|
677
|
+
rbld_print.progress_tick
|
678
|
+
|
679
|
+
names = NameFactory.new( env )
|
680
|
+
new_img = env.modification_container.commit( names.rerun )
|
681
|
+
|
682
|
+
rbld_print.progress_tick
|
683
|
+
|
684
|
+
#Remove old re-run image in case it became dangling
|
685
|
+
@cache.dangling.each(&:remove)
|
686
|
+
|
687
|
+
rbld_print.progress_end
|
688
|
+
|
689
|
+
@cache.refresh!
|
690
|
+
|
691
|
+
run_env( @cache.get(env), cmd, rerun: true )
|
692
|
+
end
|
693
|
+
|
694
|
+
def existing_env(name)
|
695
|
+
env = @cache.get(name)
|
696
|
+
raise EnvironmentNotKnown, name.full unless env
|
697
|
+
env
|
698
|
+
end
|
699
|
+
|
700
|
+
def unmodified_env(name)
|
701
|
+
env = existing_env( name )
|
702
|
+
raise EnvironmentIsModified if env.modified?
|
703
|
+
env
|
704
|
+
end
|
705
|
+
|
706
|
+
def nonexisting_env(name)
|
707
|
+
raise EnvironmentAlreadyExists, name.full if @cache.get(name)
|
708
|
+
end
|
709
|
+
end
|
710
|
+
end
|