fpm-fry 0.1.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (40) hide show
  1. checksums.yaml +7 -0
  2. data/bin/fpm-fry +10 -0
  3. data/lib/cabin/nice_output.rb +70 -0
  4. data/lib/fpm/fry/block_enumerator.rb +25 -0
  5. data/lib/fpm/fry/build_output_parser.rb +22 -0
  6. data/lib/fpm/fry/client.rb +162 -0
  7. data/lib/fpm/fry/command/cook.rb +370 -0
  8. data/lib/fpm/fry/command.rb +90 -0
  9. data/lib/fpm/fry/detector.rb +109 -0
  10. data/lib/fpm/fry/docker_file.rb +149 -0
  11. data/lib/fpm/fry/joined_io.rb +63 -0
  12. data/lib/fpm/fry/os_db.rb +35 -0
  13. data/lib/fpm/fry/plugin/alternatives.rb +90 -0
  14. data/lib/fpm/fry/plugin/edit_staging.rb +66 -0
  15. data/lib/fpm/fry/plugin/exclude.rb +18 -0
  16. data/lib/fpm/fry/plugin/init.rb +53 -0
  17. data/lib/fpm/fry/plugin/platforms.rb +10 -0
  18. data/lib/fpm/fry/plugin/script_helper.rb +176 -0
  19. data/lib/fpm/fry/plugin/service.rb +100 -0
  20. data/lib/fpm/fry/plugin.rb +3 -0
  21. data/lib/fpm/fry/recipe/builder.rb +267 -0
  22. data/lib/fpm/fry/recipe.rb +141 -0
  23. data/lib/fpm/fry/source/dir.rb +56 -0
  24. data/lib/fpm/fry/source/git.rb +90 -0
  25. data/lib/fpm/fry/source/package.rb +202 -0
  26. data/lib/fpm/fry/source/patched.rb +118 -0
  27. data/lib/fpm/fry/source.rb +47 -0
  28. data/lib/fpm/fry/stream_parser.rb +98 -0
  29. data/lib/fpm/fry/tar.rb +71 -0
  30. data/lib/fpm/fry/templates/debian/after_install.erb +9 -0
  31. data/lib/fpm/fry/templates/debian/before_install.erb +13 -0
  32. data/lib/fpm/fry/templates/debian/before_remove.erb +13 -0
  33. data/lib/fpm/fry/templates/redhat/after_install.erb +2 -0
  34. data/lib/fpm/fry/templates/redhat/before_install.erb +6 -0
  35. data/lib/fpm/fry/templates/redhat/before_remove.erb +6 -0
  36. data/lib/fpm/fry/templates/sysv.erb +125 -0
  37. data/lib/fpm/fry/templates/upstart.erb +15 -0
  38. data/lib/fpm/fry/ui.rb +12 -0
  39. data/lib/fpm/package/docker.rb +186 -0
  40. metadata +111 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 6d10781e6b5740b80f68e6a7130acd844c2b1d5c
4
+ data.tar.gz: 5dc23f36f547bd2359b433424ae8ef091dd2cf53
5
+ SHA512:
6
+ metadata.gz: ba95b38a5652f87439105db3c7430505695e5429ef90fdfadcb1b6f517556259c8add886a2c0b0cca72e0d7c7683a4584883d3e1263266ad5f4e162fa911b9f0
7
+ data.tar.gz: 9ded152eb45fbab21bdef04a651bee26689f0dd8a45a10d096ece7fe5cbc646f727fbc082e353f49bd62b6a303887d6ab86b6bb0e245625667d0529779f39f3e
data/bin/fpm-fry ADDED
@@ -0,0 +1,10 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "rubygems"
4
+ $: << File.expand_path(File.join( "..", "lib"),File.dirname(__FILE__))
5
+ require "fpm"
6
+ require "fpm/command"
7
+ require "fpm/package/docker"
8
+ require "fpm/fry/command"
9
+
10
+ exit(FPM::Fry::Command.run || 0)
@@ -0,0 +1,70 @@
1
+ require 'cabin'
2
+ class Cabin::NiceOutput
3
+
4
+ CODEMAP = {
5
+ :normal => "\e[0m",
6
+ :red => "\e[1;31m",
7
+ :green => "\e[1;32m",
8
+ :yellow => "\e[0;33m",
9
+ :white => "\e[0;37m"
10
+ }
11
+
12
+ DIM_CODEMAP = {
13
+ red: "\e[0;31m",
14
+ green: "\e[0;32m",
15
+ white: "\e[1;30m",
16
+ yellow: "\e[33m"
17
+ }
18
+
19
+ LEVELMAP = {
20
+ :fatal => :red,
21
+ :error => :red,
22
+ :warn => :yellow,
23
+ :info => :green,
24
+ :debug => :white,
25
+ }
26
+
27
+ attr :io
28
+
29
+ def initialize(io)
30
+ @io = io
31
+ end
32
+
33
+ def <<(event)
34
+ data = event.clone
35
+ data.delete(:line)
36
+ data.delete(:file)
37
+ level = data.delete(:level) || :normal
38
+ data.delete(:message)
39
+ ts = data.delete(:timestamp)
40
+
41
+ color = data.delete(:color)
42
+ # :bold is expected to be truthy
43
+ bold = data.delete(:bold) ? :bold : nil
44
+
45
+ backtrace = data.delete(:backtrace)
46
+
47
+ # Make 'error' and other log levels have color
48
+ if color.nil?
49
+ color = LEVELMAP[level]
50
+ end
51
+
52
+ message = [event[:level] ? '====> ' : ' ',event[:message]]
53
+ message.unshift(CODEMAP[color.to_sym]) if !color.nil?
54
+ message << DIM_CODEMAP[color] if !color.nil?
55
+ if data.any?
56
+ message << "\n" << pp(data)
57
+ end
58
+ if backtrace
59
+ message << "\n\t--backtrace---------------\n\t" << backtrace.join("\n\t")
60
+ end
61
+ message << CODEMAP[:normal] if !color.nil?
62
+ @io.puts(message.join(""))
63
+ @io.flush
64
+ end
65
+
66
+ def pp(hash)
67
+ hash.map{|k,v| ' '+k.to_s + ": " + v.inspect }.join("\n")
68
+ end
69
+
70
+ end
@@ -0,0 +1,25 @@
1
+ module FPM; module Fry
2
+ class BlockEnumerator < Struct.new(:io, :blocksize)
3
+ include Enumerable
4
+
5
+ def initialize(_, blocksize = 128)
6
+ super
7
+ end
8
+
9
+ def each
10
+ return to_enum unless block_given?
11
+ # Reading bigger chunks is far more efficient that eaching over the
12
+ while chunk = io.read(blocksize)
13
+ yield chunk
14
+ end
15
+ end
16
+
17
+ def call
18
+ while x = io.read(blocksize)
19
+ next if x == ""
20
+ return x
21
+ end
22
+ return ""
23
+ end
24
+ end
25
+ end ; end
@@ -0,0 +1,22 @@
1
+ require 'json'
2
+ module FPM; module Fry
3
+ class BuildOutputParser < Struct.new(:out)
4
+
5
+ attr :images
6
+
7
+ def initialize(*_)
8
+ super
9
+ @images = []
10
+ end
11
+
12
+ def call(chunk, *_)
13
+ json = JSON.parse(chunk)
14
+ stream = json['stream']
15
+ if /\ASuccessfully built (\w+)\Z/.match(stream)
16
+ images << $1
17
+ end
18
+ out << stream
19
+ end
20
+
21
+ end
22
+ end ; end
@@ -0,0 +1,162 @@
1
+ require 'excon'
2
+ require 'rubygems/package'
3
+ require 'json'
4
+ require 'fileutils'
5
+ require 'forwardable'
6
+ require 'fpm/fry/tar'
7
+
8
+ module FPM; module Fry; end ; end
9
+
10
+ class FPM::Fry::Client
11
+
12
+ class LogInstrumentor < Struct.new(:logger)
13
+
14
+ def instrument(event, data = {})
15
+ if block_given?
16
+ logger.debug('Requesting HTTP', filtered(data))
17
+ r = yield
18
+ return r
19
+ else
20
+ logger.debug('Getting HTTP response', filtered(data))
21
+ end
22
+ end
23
+
24
+ def filtered(data)
25
+ filtered = {}
26
+ filtered[:path] = data[:path] if data[:path]
27
+ filtered[:verb] = data[:method] if data[:method]
28
+ filtered[:status] = data[:status] if data[:status]
29
+ filtered[:body] = data[:body][0..500] if data[:body]
30
+ filtered[:headers] = data[:headers]
31
+ return filtered
32
+ end
33
+
34
+ end
35
+
36
+ class FileNotFound < StandardError
37
+ end
38
+
39
+ extend Forwardable
40
+ def_delegators :agent, :post, :get, :delete
41
+
42
+ attr :docker_url, :logger, :tls
43
+
44
+ def initialize(options = {})
45
+ @docker_url = options.fetch(:docker_url){ self.class.docker_url }
46
+ @logger = options[:logger]
47
+ if @logger.nil?
48
+ @logger = Cabin::Channel.get
49
+ end
50
+ if options[:tls].nil? ? docker_url =~ %r!(\Ahttps://|:2376\z)! : options[:tls]
51
+ # enable tls
52
+ @tls = {
53
+ client_cert: File.join(self.class.docker_cert_path,'cert.pem'),
54
+ client_key: File.join(self.class.docker_cert_path, 'key.pem'),
55
+ ssl_ca_file: File.join(self.class.docker_cert_path, 'ca.pem'),
56
+ ssl_verify_peer: options.fetch(:tlsverify){ false }
57
+ }
58
+ [:client_cert, :client_key, :ssl_ca_file].each do |k|
59
+ if !File.exists?(@tls[k])
60
+ raise ArgumentError.new("#{k} #{@tls[k]} doesn't exist. Did you set DOCKER_CERT_PATH correctly?")
61
+ end
62
+ end
63
+ else
64
+ @tls = {}
65
+ end
66
+ end
67
+
68
+ def server_version
69
+ @server_version ||= begin
70
+ res = agent.get(
71
+ expects: [200],
72
+ path: '/version'
73
+ )
74
+ JSON.parse(res.body)
75
+ end
76
+ end
77
+
78
+ def self.docker_cert_path
79
+ ENV.fetch('DOCKER_CERT_PATH',File.join(Dir.home, '.docker'))
80
+ end
81
+
82
+ def self.docker_url
83
+ ENV.fetch('DOCKER_HOST'.freeze, 'unix:///var/run/docker.sock')
84
+ end
85
+
86
+ def tls?
87
+ tls.any?
88
+ end
89
+
90
+
91
+ def url(*path)
92
+ ['', "v"+server_version['ApiVersion'],*path].join('/')
93
+ end
94
+
95
+ def read(name, resource)
96
+ return to_enum(:read, name, resource) unless block_given?
97
+ body = JSON.generate({'Resource' => resource})
98
+ res = agent.post(
99
+ path: url('containers',name,'copy'),
100
+ headers: { 'Content-Type' => 'application/json' },
101
+ body: body,
102
+ expects: [200,500]
103
+ )
104
+ if res.status == 500
105
+ raise FileNotFound, "File #{resource.inspect} not found: #{res.body}"
106
+ end
107
+ sio = StringIO.new(res.body)
108
+ tar = ::Gem::Package::TarReader.new( sio )
109
+ tar.each do |entry|
110
+ yield entry
111
+ end
112
+ end
113
+
114
+ def copy(name, resource, map, options = {})
115
+ ex = FPM::Fry::Tar::Extractor.new(logger: @logger)
116
+ base = File.dirname(resource)
117
+ read(name, resource) do | entry |
118
+ file = File.join(base, entry.full_name).chomp('/')
119
+ file = file.sub(%r"\A\./",'')
120
+ to = map[file]
121
+ next unless to
122
+ @logger.debug("Copy",name: file, to: to)
123
+ ex.extract_entry(to, entry, options)
124
+ end
125
+ end
126
+
127
+ def changes(name)
128
+ res = agent.get(path: url('containers',name,'changes'))
129
+ raise res.reason if res.status != 200
130
+ return JSON.parse(res.body)
131
+ end
132
+
133
+ def agent
134
+ @agent ||= agent_for(docker_url, tls)
135
+ end
136
+
137
+ def broken_symlinks?
138
+ return true
139
+ end
140
+
141
+ def agent_for( uri, tls )
142
+ proto, address = uri.split('://',2)
143
+ options = {
144
+ instrumentor: LogInstrumentor.new(logger),
145
+ read_timeout: 10000
146
+ }.merge( tls )
147
+ case(proto)
148
+ when 'unix'
149
+ uri = "unix:///"
150
+ options[:socket] = address
151
+ when 'tcp'
152
+ if tls.any?
153
+ return agent_for("https://#{address}", tls)
154
+ else
155
+ return agent_for("http://#{address}", tls)
156
+ end
157
+ when 'http', 'https'
158
+ end
159
+ logger.debug("Creating Agent", options.merge(uri: uri))
160
+ return Excon.new(uri, options)
161
+ end
162
+ end
@@ -0,0 +1,370 @@
1
+ require 'fpm/fry/command'
2
+ module FPM; module Fry
3
+ class Command::Cook < Command
4
+
5
+ option '--distribution', 'distribution', 'Distribution like ubuntu-12.04'
6
+ option '--keep', :flag, 'Keep the container after build'
7
+ option '--overwrite', :flag, 'Overwrite package', default: true
8
+
9
+ UPDATE_VALUES = ['auto','never','always']
10
+ option '--update',"<#{UPDATE_VALUES.join('|')}>", 'Update image before installing packages ( only apt currently )',attribute_name: 'update', default: 'auto' do |value|
11
+ if !UPDATE_VALUES.include? value
12
+ raise "Unknown value for --update: #{value.inspect}\nPossible values are #{UPDATE_VALUES.join(', ')}"
13
+ else
14
+ value
15
+ end
16
+ end
17
+
18
+ parameter 'image', 'Docker image to build from'
19
+ parameter '[recipe]', 'Recipe file to cook', default: 'recipe.rb'
20
+
21
+ attr :ui
22
+ extend Forwardable
23
+ def_delegators :ui, :out, :err, :logger, :tmpdir
24
+
25
+ def initialize(invocation_path, ctx = {}, parent_attribute_values = {})
26
+ @tls = nil
27
+ require 'digest'
28
+ require 'fileutils'
29
+ require 'fpm/fry/recipe'
30
+ require 'fpm/fry/recipe/builder'
31
+ require 'fpm/fry/detector'
32
+ require 'fpm/fry/docker_file'
33
+ require 'fpm/fry/stream_parser'
34
+ require 'fpm/fry/os_db'
35
+ require 'fpm/fry/block_enumerator'
36
+ require 'fpm/fry/build_output_parser'
37
+ super
38
+ @ui = ctx.fetch(:ui){ UI.new }
39
+ if debug?
40
+ ui.logger.level = :debug
41
+ end
42
+ end
43
+
44
+ def detector
45
+ @detector || begin
46
+ if distribution
47
+ d = Detector::String.new(distribution)
48
+ else
49
+ d = Detector::Image.new(client, image)
50
+ end
51
+ self.detector=d
52
+ end
53
+ end
54
+
55
+ def detector=(d)
56
+ begin
57
+ unless d.detect!
58
+ raise "Unable to detect distribution from given image"
59
+ end
60
+ rescue Excon::Errors::NotFound
61
+ raise "Image not found"
62
+ end
63
+ @detector = d
64
+ end
65
+
66
+ def flavour
67
+ @flavour ||= OsDb.fetch(detector.distribution,{flavour: "unknown"})[:flavour]
68
+ end
69
+ attr_writer :flavour
70
+
71
+ def output_class
72
+ @output_class ||= begin
73
+ logger.info("Autodetecting package type",flavour: flavour)
74
+ case(flavour)
75
+ when 'debian'
76
+ require 'fpm/package/deb'
77
+ FPM::Package::Deb
78
+ when 'redhat'
79
+ require 'fpm/package/rpm'
80
+ FPM::Package::RPM
81
+ else
82
+ raise "Cannot auto-detect package type."
83
+ end
84
+ end
85
+ end
86
+ attr_writer :output_class
87
+
88
+ def builder
89
+ @builder ||= begin
90
+ vars = {
91
+ distribution: detector.distribution,
92
+ distribution_version: detector.version,
93
+ flavour: flavour
94
+ }
95
+ logger.info("Loading recipe",variables: vars, recipe: recipe)
96
+ b = Recipe::Builder.new(vars, Recipe.new, logger: ui.logger)
97
+ b.load_file( recipe )
98
+ b
99
+ end
100
+ end
101
+ attr_writer :builder
102
+
103
+ def cache
104
+ @cache ||= builder.recipe.source.build_cache(tmpdir)
105
+ end
106
+ attr_writer :cache
107
+
108
+ def lint_output_class!
109
+
110
+ end
111
+
112
+ def lint_recipe_file!
113
+ File.exists?(recipe) || raise(Recipe::NotFound)
114
+ end
115
+
116
+ def lint_recipe!
117
+ problems = builder.recipe.lint
118
+ if problems.any?
119
+ problems.each do |p|
120
+ logger.error(p)
121
+ end
122
+ raise
123
+ end
124
+ end
125
+
126
+ def image_id
127
+ @image_id ||= begin
128
+ res = client.get(
129
+ expects: [200],
130
+ path: client.url("images/#{image}/json")
131
+ )
132
+ body = JSON.parse(res.body)
133
+ body.fetch('id'){ body.fetch('Id') }
134
+ end
135
+ end
136
+ attr_writer :image_id
137
+
138
+ def build_image
139
+ @build_image ||= begin
140
+ sum = Digest::SHA256.hexdigest( image_id + "\0" + cache.cachekey )
141
+ cachetag = "fpm-fry:#{sum[0..30]}"
142
+ res = client.get(
143
+ expects: [200,404],
144
+ path: client.url("images/#{cachetag}/json")
145
+ )
146
+ if res.status == 404
147
+ df = DockerFile::Source.new(builder.variables.merge(image: image_id),cache)
148
+ client.post(
149
+ headers: {
150
+ 'Content-Type'=>'application/tar'
151
+ },
152
+ expects: [200],
153
+ path: client.url("build?rm=1&t=#{cachetag}"),
154
+ request_block: BlockEnumerator.new(df.tar_io)
155
+ )
156
+ end
157
+
158
+ df = DockerFile::Build.new(cachetag, builder.variables.dup,builder.recipe, update: update?)
159
+ parser = BuildOutputParser.new(out)
160
+ res = client.post(
161
+ headers: {
162
+ 'Content-Type'=>'application/tar'
163
+ },
164
+ expects: [200],
165
+ path: client.url('build?rm=1'),
166
+ request_block: BlockEnumerator.new(df.tar_io),
167
+ response_block: parser
168
+ )
169
+ if parser.images.none?
170
+ raise "Unable to detect build image"
171
+ end
172
+ image = parser.images.last
173
+ logger.debug("Detected build image", image: image)
174
+ image
175
+ end
176
+ end
177
+ attr_writer :build_image
178
+
179
+ def update?
180
+ if flavour == 'debian'
181
+ case(update)
182
+ when 'auto'
183
+ body = JSON.generate({"Image" => image, "Cmd" => "exit 0"})
184
+ res = client.post( path: client.url('containers','create'),
185
+ headers: {'Content-Type' => 'application/json'},
186
+ body: body,
187
+ expects: [201]
188
+ )
189
+ body = JSON.parse(res.body)
190
+ container = body.fetch('Id')
191
+ begin
192
+ client.read( container, '/var/lib/apt/lists') do |file|
193
+ next if file.header.name == 'lists/'
194
+ logger.info("/var/lib/apt/lists is not empty, no update is required ( force update with --apt-update=always )")
195
+ return false
196
+ end
197
+ ensure
198
+ client.delete(path: client.url('containers',container))
199
+ end
200
+ logger.info("/var/lib/apt/lists is empty, update is required ( disable update with --apt-update=never )")
201
+ return true
202
+ when 'always'
203
+ return true
204
+ when 'never'
205
+ return false
206
+ end
207
+ else
208
+ return false
209
+ end
210
+ end
211
+
212
+ def build!
213
+ res = client.post(
214
+ headers: {
215
+ 'Content-Type' => 'application/json'
216
+ },
217
+ path: client.url('containers','create'),
218
+ expects: [201],
219
+ body: JSON.generate({"Image" => build_image})
220
+ )
221
+
222
+ body = JSON.parse(res.body)
223
+ container = body['Id']
224
+ begin
225
+ client.post(
226
+ headers: {
227
+ 'Content-Type' => 'application/json'
228
+ },
229
+ path: client.url('containers',container,'start'),
230
+ expects: [204],
231
+ body: JSON.generate({})
232
+ )
233
+
234
+ client.post(
235
+ path: client.url('containers',container,'attach?stderr=1&stdout=1&stream=1'),
236
+ body: '',
237
+ expects: [200],
238
+ middlewares: [
239
+ StreamParser.new(out,err),
240
+ Excon::Middleware::Expects,
241
+ Excon::Middleware::Instrumentor,
242
+ Excon::Middleware::Mock
243
+ ]
244
+ )
245
+
246
+ res = client.post(
247
+ path: client.url('containers',container,'wait'),
248
+ expects: [200],
249
+ body: ''
250
+ )
251
+ json = JSON.parse(res.body)
252
+ if json["StatusCode"] != 0
253
+ raise "Build failed"
254
+ end
255
+ return yield container
256
+ ensure
257
+ unless keep?
258
+ client.delete(path: client.url('containers',container))
259
+ end
260
+ end
261
+ end
262
+
263
+ def input_package(container)
264
+ input = FPM::Package::Docker.new(logger: logger, client: client)
265
+ builder.recipe.apply_input(input)
266
+ begin
267
+ input.input(container)
268
+ return yield(input)
269
+ ensure
270
+ input.cleanup_staging
271
+ input.cleanup_build
272
+ end
273
+ end
274
+
275
+ def write_output!(output)
276
+ package_file = File.expand_path(output.to_s(nil))
277
+ FileUtils.mkdir_p(File.dirname(package_file))
278
+ tmp_package_file = package_file + '.tmp'
279
+ begin
280
+ FileUtils.rm_rf tmp_package_file
281
+ rescue Errno::ENOENT
282
+ end
283
+
284
+ output.output(tmp_package_file)
285
+
286
+ begin
287
+ FileUtils.rm_rf package_file
288
+ rescue Errno::ENOENT
289
+ end
290
+ File.rename tmp_package_file, package_file
291
+
292
+ logger.info("Created package", :path => package_file)
293
+ end
294
+
295
+ def packages
296
+ dir_map = []
297
+ out_map = {}
298
+
299
+ package_map = builder.recipe.packages.map do | package |
300
+ output = output_class.new
301
+ output.instance_variable_set(:@logger,logger)
302
+ package.files.each do | pattern |
303
+ dir_map << [ pattern, output.staging_path ]
304
+ end
305
+ out_map[ output ] = package
306
+ end
307
+
308
+ dir_map = Hash[ dir_map.reverse ]
309
+
310
+ yield dir_map
311
+
312
+ out_map.each do |output, package|
313
+ package.apply_output(output)
314
+ end
315
+
316
+ out_map.each do |output, _|
317
+ write_output!(output)
318
+ end
319
+
320
+ ensure
321
+
322
+ out_map.each do |output, _|
323
+ output.cleanup_staging
324
+ output.cleanup_build
325
+ end
326
+
327
+ end
328
+
329
+
330
+ public
331
+
332
+ def execute
333
+ # force some eager loading
334
+ lint_recipe_file!
335
+ detector
336
+ flavour
337
+ output_class
338
+ lint_output_class!
339
+ builder
340
+ lint_recipe!
341
+ cache
342
+
343
+ image_id
344
+ build_image
345
+
346
+ packages do | dir_map |
347
+
348
+ build! do |container|
349
+ input_package(container) do |input|
350
+ input.split( container, dir_map )
351
+ end
352
+ end
353
+
354
+ end
355
+
356
+ return 0
357
+ rescue Recipe::NotFound => e
358
+ logger.error("Recipe not found", recipe: recipe, exeception: e)
359
+ return 1
360
+ rescue => e
361
+ logger.error(e)
362
+ return 1
363
+ end
364
+
365
+ end
366
+
367
+ class Command
368
+ subcommand 'cook', 'Cooks a package', Cook
369
+ end
370
+ end ; end