fpm-fry 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +7 -0
  2. data/bin/fpm-fry +10 -0
  3. data/lib/cabin/nice_output.rb +70 -0
  4. data/lib/fpm/fry/block_enumerator.rb +25 -0
  5. data/lib/fpm/fry/build_output_parser.rb +22 -0
  6. data/lib/fpm/fry/client.rb +162 -0
  7. data/lib/fpm/fry/command/cook.rb +370 -0
  8. data/lib/fpm/fry/command.rb +90 -0
  9. data/lib/fpm/fry/detector.rb +109 -0
  10. data/lib/fpm/fry/docker_file.rb +149 -0
  11. data/lib/fpm/fry/joined_io.rb +63 -0
  12. data/lib/fpm/fry/os_db.rb +35 -0
  13. data/lib/fpm/fry/plugin/alternatives.rb +90 -0
  14. data/lib/fpm/fry/plugin/edit_staging.rb +66 -0
  15. data/lib/fpm/fry/plugin/exclude.rb +18 -0
  16. data/lib/fpm/fry/plugin/init.rb +53 -0
  17. data/lib/fpm/fry/plugin/platforms.rb +10 -0
  18. data/lib/fpm/fry/plugin/script_helper.rb +176 -0
  19. data/lib/fpm/fry/plugin/service.rb +100 -0
  20. data/lib/fpm/fry/plugin.rb +3 -0
  21. data/lib/fpm/fry/recipe/builder.rb +267 -0
  22. data/lib/fpm/fry/recipe.rb +141 -0
  23. data/lib/fpm/fry/source/dir.rb +56 -0
  24. data/lib/fpm/fry/source/git.rb +90 -0
  25. data/lib/fpm/fry/source/package.rb +202 -0
  26. data/lib/fpm/fry/source/patched.rb +118 -0
  27. data/lib/fpm/fry/source.rb +47 -0
  28. data/lib/fpm/fry/stream_parser.rb +98 -0
  29. data/lib/fpm/fry/tar.rb +71 -0
  30. data/lib/fpm/fry/templates/debian/after_install.erb +9 -0
  31. data/lib/fpm/fry/templates/debian/before_install.erb +13 -0
  32. data/lib/fpm/fry/templates/debian/before_remove.erb +13 -0
  33. data/lib/fpm/fry/templates/redhat/after_install.erb +2 -0
  34. data/lib/fpm/fry/templates/redhat/before_install.erb +6 -0
  35. data/lib/fpm/fry/templates/redhat/before_remove.erb +6 -0
  36. data/lib/fpm/fry/templates/sysv.erb +125 -0
  37. data/lib/fpm/fry/templates/upstart.erb +15 -0
  38. data/lib/fpm/fry/ui.rb +12 -0
  39. data/lib/fpm/package/docker.rb +186 -0
  40. metadata +111 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 6d10781e6b5740b80f68e6a7130acd844c2b1d5c
4
+ data.tar.gz: 5dc23f36f547bd2359b433424ae8ef091dd2cf53
5
+ SHA512:
6
+ metadata.gz: ba95b38a5652f87439105db3c7430505695e5429ef90fdfadcb1b6f517556259c8add886a2c0b0cca72e0d7c7683a4584883d3e1263266ad5f4e162fa911b9f0
7
+ data.tar.gz: 9ded152eb45fbab21bdef04a651bee26689f0dd8a45a10d096ece7fe5cbc646f727fbc082e353f49bd62b6a303887d6ab86b6bb0e245625667d0529779f39f3e
data/bin/fpm-fry ADDED
@@ -0,0 +1,10 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "rubygems"
4
+ $: << File.expand_path(File.join( "..", "lib"),File.dirname(__FILE__))
5
+ require "fpm"
6
+ require "fpm/command"
7
+ require "fpm/package/docker"
8
+ require "fpm/fry/command"
9
+
10
+ exit(FPM::Fry::Command.run || 0)
@@ -0,0 +1,70 @@
1
+ require 'cabin'
2
+ class Cabin::NiceOutput
3
+
4
+ CODEMAP = {
5
+ :normal => "\e[0m",
6
+ :red => "\e[1;31m",
7
+ :green => "\e[1;32m",
8
+ :yellow => "\e[0;33m",
9
+ :white => "\e[0;37m"
10
+ }
11
+
12
+ DIM_CODEMAP = {
13
+ red: "\e[0;31m",
14
+ green: "\e[0;32m",
15
+ white: "\e[1;30m",
16
+ yellow: "\e[33m"
17
+ }
18
+
19
+ LEVELMAP = {
20
+ :fatal => :red,
21
+ :error => :red,
22
+ :warn => :yellow,
23
+ :info => :green,
24
+ :debug => :white,
25
+ }
26
+
27
+ attr :io
28
+
29
+ def initialize(io)
30
+ @io = io
31
+ end
32
+
33
+ def <<(event)
34
+ data = event.clone
35
+ data.delete(:line)
36
+ data.delete(:file)
37
+ level = data.delete(:level) || :normal
38
+ data.delete(:message)
39
+ ts = data.delete(:timestamp)
40
+
41
+ color = data.delete(:color)
42
+ # :bold is expected to be truthy
43
+ bold = data.delete(:bold) ? :bold : nil
44
+
45
+ backtrace = data.delete(:backtrace)
46
+
47
+ # Make 'error' and other log levels have color
48
+ if color.nil?
49
+ color = LEVELMAP[level]
50
+ end
51
+
52
+ message = [event[:level] ? '====> ' : ' ',event[:message]]
53
+ message.unshift(CODEMAP[color.to_sym]) if !color.nil?
54
+ message << DIM_CODEMAP[color] if !color.nil?
55
+ if data.any?
56
+ message << "\n" << pp(data)
57
+ end
58
+ if backtrace
59
+ message << "\n\t--backtrace---------------\n\t" << backtrace.join("\n\t")
60
+ end
61
+ message << CODEMAP[:normal] if !color.nil?
62
+ @io.puts(message.join(""))
63
+ @io.flush
64
+ end
65
+
66
+ def pp(hash)
67
+ hash.map{|k,v| ' '+k.to_s + ": " + v.inspect }.join("\n")
68
+ end
69
+
70
+ end
@@ -0,0 +1,25 @@
1
+ module FPM; module Fry
2
+ class BlockEnumerator < Struct.new(:io, :blocksize)
3
+ include Enumerable
4
+
5
+ def initialize(_, blocksize = 128)
6
+ super
7
+ end
8
+
9
+ def each
10
+ return to_enum unless block_given?
11
+ # Reading bigger chunks is far more efficient that eaching over the
12
+ while chunk = io.read(blocksize)
13
+ yield chunk
14
+ end
15
+ end
16
+
17
+ def call
18
+ while x = io.read(blocksize)
19
+ next if x == ""
20
+ return x
21
+ end
22
+ return ""
23
+ end
24
+ end
25
+ end ; end
@@ -0,0 +1,22 @@
1
+ require 'json'
2
+ module FPM; module Fry
3
+ class BuildOutputParser < Struct.new(:out)
4
+
5
+ attr :images
6
+
7
+ def initialize(*_)
8
+ super
9
+ @images = []
10
+ end
11
+
12
+ def call(chunk, *_)
13
+ json = JSON.parse(chunk)
14
+ stream = json['stream']
15
+ if /\ASuccessfully built (\w+)\Z/.match(stream)
16
+ images << $1
17
+ end
18
+ out << stream
19
+ end
20
+
21
+ end
22
+ end ; end
@@ -0,0 +1,162 @@
1
+ require 'excon'
2
+ require 'rubygems/package'
3
+ require 'json'
4
+ require 'fileutils'
5
+ require 'forwardable'
6
+ require 'fpm/fry/tar'
7
+
8
+ module FPM; module Fry; end ; end
9
+
10
+ class FPM::Fry::Client
11
+
12
+ class LogInstrumentor < Struct.new(:logger)
13
+
14
+ def instrument(event, data = {})
15
+ if block_given?
16
+ logger.debug('Requesting HTTP', filtered(data))
17
+ r = yield
18
+ return r
19
+ else
20
+ logger.debug('Getting HTTP response', filtered(data))
21
+ end
22
+ end
23
+
24
+ def filtered(data)
25
+ filtered = {}
26
+ filtered[:path] = data[:path] if data[:path]
27
+ filtered[:verb] = data[:method] if data[:method]
28
+ filtered[:status] = data[:status] if data[:status]
29
+ filtered[:body] = data[:body][0..500] if data[:body]
30
+ filtered[:headers] = data[:headers]
31
+ return filtered
32
+ end
33
+
34
+ end
35
+
36
+ class FileNotFound < StandardError
37
+ end
38
+
39
+ extend Forwardable
40
+ def_delegators :agent, :post, :get, :delete
41
+
42
+ attr :docker_url, :logger, :tls
43
+
44
+ def initialize(options = {})
45
+ @docker_url = options.fetch(:docker_url){ self.class.docker_url }
46
+ @logger = options[:logger]
47
+ if @logger.nil?
48
+ @logger = Cabin::Channel.get
49
+ end
50
+ if options[:tls].nil? ? docker_url =~ %r!(\Ahttps://|:2376\z)! : options[:tls]
51
+ # enable tls
52
+ @tls = {
53
+ client_cert: File.join(self.class.docker_cert_path,'cert.pem'),
54
+ client_key: File.join(self.class.docker_cert_path, 'key.pem'),
55
+ ssl_ca_file: File.join(self.class.docker_cert_path, 'ca.pem'),
56
+ ssl_verify_peer: options.fetch(:tlsverify){ false }
57
+ }
58
+ [:client_cert, :client_key, :ssl_ca_file].each do |k|
59
+ if !File.exists?(@tls[k])
60
+ raise ArgumentError.new("#{k} #{@tls[k]} doesn't exist. Did you set DOCKER_CERT_PATH correctly?")
61
+ end
62
+ end
63
+ else
64
+ @tls = {}
65
+ end
66
+ end
67
+
68
+ def server_version
69
+ @server_version ||= begin
70
+ res = agent.get(
71
+ expects: [200],
72
+ path: '/version'
73
+ )
74
+ JSON.parse(res.body)
75
+ end
76
+ end
77
+
78
+ def self.docker_cert_path
79
+ ENV.fetch('DOCKER_CERT_PATH',File.join(Dir.home, '.docker'))
80
+ end
81
+
82
+ def self.docker_url
83
+ ENV.fetch('DOCKER_HOST'.freeze, 'unix:///var/run/docker.sock')
84
+ end
85
+
86
+ def tls?
87
+ tls.any?
88
+ end
89
+
90
+
91
+ def url(*path)
92
+ ['', "v"+server_version['ApiVersion'],*path].join('/')
93
+ end
94
+
95
+ def read(name, resource)
96
+ return to_enum(:read, name, resource) unless block_given?
97
+ body = JSON.generate({'Resource' => resource})
98
+ res = agent.post(
99
+ path: url('containers',name,'copy'),
100
+ headers: { 'Content-Type' => 'application/json' },
101
+ body: body,
102
+ expects: [200,500]
103
+ )
104
+ if res.status == 500
105
+ raise FileNotFound, "File #{resource.inspect} not found: #{res.body}"
106
+ end
107
+ sio = StringIO.new(res.body)
108
+ tar = ::Gem::Package::TarReader.new( sio )
109
+ tar.each do |entry|
110
+ yield entry
111
+ end
112
+ end
113
+
114
+ def copy(name, resource, map, options = {})
115
+ ex = FPM::Fry::Tar::Extractor.new(logger: @logger)
116
+ base = File.dirname(resource)
117
+ read(name, resource) do | entry |
118
+ file = File.join(base, entry.full_name).chomp('/')
119
+ file = file.sub(%r"\A\./",'')
120
+ to = map[file]
121
+ next unless to
122
+ @logger.debug("Copy",name: file, to: to)
123
+ ex.extract_entry(to, entry, options)
124
+ end
125
+ end
126
+
127
+ def changes(name)
128
+ res = agent.get(path: url('containers',name,'changes'))
129
+ raise res.reason if res.status != 200
130
+ return JSON.parse(res.body)
131
+ end
132
+
133
+ def agent
134
+ @agent ||= agent_for(docker_url, tls)
135
+ end
136
+
137
+ def broken_symlinks?
138
+ return true
139
+ end
140
+
141
+ def agent_for( uri, tls )
142
+ proto, address = uri.split('://',2)
143
+ options = {
144
+ instrumentor: LogInstrumentor.new(logger),
145
+ read_timeout: 10000
146
+ }.merge( tls )
147
+ case(proto)
148
+ when 'unix'
149
+ uri = "unix:///"
150
+ options[:socket] = address
151
+ when 'tcp'
152
+ if tls.any?
153
+ return agent_for("https://#{address}", tls)
154
+ else
155
+ return agent_for("http://#{address}", tls)
156
+ end
157
+ when 'http', 'https'
158
+ end
159
+ logger.debug("Creating Agent", options.merge(uri: uri))
160
+ return Excon.new(uri, options)
161
+ end
162
+ end
@@ -0,0 +1,370 @@
1
+ require 'fpm/fry/command'
2
+ module FPM; module Fry
3
+ class Command::Cook < Command
4
+
5
+ option '--distribution', 'distribution', 'Distribution like ubuntu-12.04'
6
+ option '--keep', :flag, 'Keep the container after build'
7
+ option '--overwrite', :flag, 'Overwrite package', default: true
8
+
9
+ UPDATE_VALUES = ['auto','never','always']
10
+ option '--update',"<#{UPDATE_VALUES.join('|')}>", 'Update image before installing packages ( only apt currently )',attribute_name: 'update', default: 'auto' do |value|
11
+ if !UPDATE_VALUES.include? value
12
+ raise "Unknown value for --update: #{value.inspect}\nPossible values are #{UPDATE_VALUES.join(', ')}"
13
+ else
14
+ value
15
+ end
16
+ end
17
+
18
+ parameter 'image', 'Docker image to build from'
19
+ parameter '[recipe]', 'Recipe file to cook', default: 'recipe.rb'
20
+
21
+ attr :ui
22
+ extend Forwardable
23
+ def_delegators :ui, :out, :err, :logger, :tmpdir
24
+
25
+ def initialize(invocation_path, ctx = {}, parent_attribute_values = {})
26
+ @tls = nil
27
+ require 'digest'
28
+ require 'fileutils'
29
+ require 'fpm/fry/recipe'
30
+ require 'fpm/fry/recipe/builder'
31
+ require 'fpm/fry/detector'
32
+ require 'fpm/fry/docker_file'
33
+ require 'fpm/fry/stream_parser'
34
+ require 'fpm/fry/os_db'
35
+ require 'fpm/fry/block_enumerator'
36
+ require 'fpm/fry/build_output_parser'
37
+ super
38
+ @ui = ctx.fetch(:ui){ UI.new }
39
+ if debug?
40
+ ui.logger.level = :debug
41
+ end
42
+ end
43
+
44
+ def detector
45
+ @detector || begin
46
+ if distribution
47
+ d = Detector::String.new(distribution)
48
+ else
49
+ d = Detector::Image.new(client, image)
50
+ end
51
+ self.detector=d
52
+ end
53
+ end
54
+
55
+ def detector=(d)
56
+ begin
57
+ unless d.detect!
58
+ raise "Unable to detect distribution from given image"
59
+ end
60
+ rescue Excon::Errors::NotFound
61
+ raise "Image not found"
62
+ end
63
+ @detector = d
64
+ end
65
+
66
+ def flavour
67
+ @flavour ||= OsDb.fetch(detector.distribution,{flavour: "unknown"})[:flavour]
68
+ end
69
+ attr_writer :flavour
70
+
71
+ def output_class
72
+ @output_class ||= begin
73
+ logger.info("Autodetecting package type",flavour: flavour)
74
+ case(flavour)
75
+ when 'debian'
76
+ require 'fpm/package/deb'
77
+ FPM::Package::Deb
78
+ when 'redhat'
79
+ require 'fpm/package/rpm'
80
+ FPM::Package::RPM
81
+ else
82
+ raise "Cannot auto-detect package type."
83
+ end
84
+ end
85
+ end
86
+ attr_writer :output_class
87
+
88
+ def builder
89
+ @builder ||= begin
90
+ vars = {
91
+ distribution: detector.distribution,
92
+ distribution_version: detector.version,
93
+ flavour: flavour
94
+ }
95
+ logger.info("Loading recipe",variables: vars, recipe: recipe)
96
+ b = Recipe::Builder.new(vars, Recipe.new, logger: ui.logger)
97
+ b.load_file( recipe )
98
+ b
99
+ end
100
+ end
101
+ attr_writer :builder
102
+
103
+ def cache
104
+ @cache ||= builder.recipe.source.build_cache(tmpdir)
105
+ end
106
+ attr_writer :cache
107
+
108
+ def lint_output_class!
109
+
110
+ end
111
+
112
+ def lint_recipe_file!
113
+ File.exists?(recipe) || raise(Recipe::NotFound)
114
+ end
115
+
116
+ def lint_recipe!
117
+ problems = builder.recipe.lint
118
+ if problems.any?
119
+ problems.each do |p|
120
+ logger.error(p)
121
+ end
122
+ raise
123
+ end
124
+ end
125
+
126
+ def image_id
127
+ @image_id ||= begin
128
+ res = client.get(
129
+ expects: [200],
130
+ path: client.url("images/#{image}/json")
131
+ )
132
+ body = JSON.parse(res.body)
133
+ body.fetch('id'){ body.fetch('Id') }
134
+ end
135
+ end
136
+ attr_writer :image_id
137
+
138
+ def build_image
139
+ @build_image ||= begin
140
+ sum = Digest::SHA256.hexdigest( image_id + "\0" + cache.cachekey )
141
+ cachetag = "fpm-fry:#{sum[0..30]}"
142
+ res = client.get(
143
+ expects: [200,404],
144
+ path: client.url("images/#{cachetag}/json")
145
+ )
146
+ if res.status == 404
147
+ df = DockerFile::Source.new(builder.variables.merge(image: image_id),cache)
148
+ client.post(
149
+ headers: {
150
+ 'Content-Type'=>'application/tar'
151
+ },
152
+ expects: [200],
153
+ path: client.url("build?rm=1&t=#{cachetag}"),
154
+ request_block: BlockEnumerator.new(df.tar_io)
155
+ )
156
+ end
157
+
158
+ df = DockerFile::Build.new(cachetag, builder.variables.dup,builder.recipe, update: update?)
159
+ parser = BuildOutputParser.new(out)
160
+ res = client.post(
161
+ headers: {
162
+ 'Content-Type'=>'application/tar'
163
+ },
164
+ expects: [200],
165
+ path: client.url('build?rm=1'),
166
+ request_block: BlockEnumerator.new(df.tar_io),
167
+ response_block: parser
168
+ )
169
+ if parser.images.none?
170
+ raise "Unable to detect build image"
171
+ end
172
+ image = parser.images.last
173
+ logger.debug("Detected build image", image: image)
174
+ image
175
+ end
176
+ end
177
+ attr_writer :build_image
178
+
179
+ def update?
180
+ if flavour == 'debian'
181
+ case(update)
182
+ when 'auto'
183
+ body = JSON.generate({"Image" => image, "Cmd" => "exit 0"})
184
+ res = client.post( path: client.url('containers','create'),
185
+ headers: {'Content-Type' => 'application/json'},
186
+ body: body,
187
+ expects: [201]
188
+ )
189
+ body = JSON.parse(res.body)
190
+ container = body.fetch('Id')
191
+ begin
192
+ client.read( container, '/var/lib/apt/lists') do |file|
193
+ next if file.header.name == 'lists/'
194
+ logger.info("/var/lib/apt/lists is not empty, no update is required ( force update with --apt-update=always )")
195
+ return false
196
+ end
197
+ ensure
198
+ client.delete(path: client.url('containers',container))
199
+ end
200
+ logger.info("/var/lib/apt/lists is empty, update is required ( disable update with --apt-update=never )")
201
+ return true
202
+ when 'always'
203
+ return true
204
+ when 'never'
205
+ return false
206
+ end
207
+ else
208
+ return false
209
+ end
210
+ end
211
+
212
+ def build!
213
+ res = client.post(
214
+ headers: {
215
+ 'Content-Type' => 'application/json'
216
+ },
217
+ path: client.url('containers','create'),
218
+ expects: [201],
219
+ body: JSON.generate({"Image" => build_image})
220
+ )
221
+
222
+ body = JSON.parse(res.body)
223
+ container = body['Id']
224
+ begin
225
+ client.post(
226
+ headers: {
227
+ 'Content-Type' => 'application/json'
228
+ },
229
+ path: client.url('containers',container,'start'),
230
+ expects: [204],
231
+ body: JSON.generate({})
232
+ )
233
+
234
+ client.post(
235
+ path: client.url('containers',container,'attach?stderr=1&stdout=1&stream=1'),
236
+ body: '',
237
+ expects: [200],
238
+ middlewares: [
239
+ StreamParser.new(out,err),
240
+ Excon::Middleware::Expects,
241
+ Excon::Middleware::Instrumentor,
242
+ Excon::Middleware::Mock
243
+ ]
244
+ )
245
+
246
+ res = client.post(
247
+ path: client.url('containers',container,'wait'),
248
+ expects: [200],
249
+ body: ''
250
+ )
251
+ json = JSON.parse(res.body)
252
+ if json["StatusCode"] != 0
253
+ raise "Build failed"
254
+ end
255
+ return yield container
256
+ ensure
257
+ unless keep?
258
+ client.delete(path: client.url('containers',container))
259
+ end
260
+ end
261
+ end
262
+
263
+ def input_package(container)
264
+ input = FPM::Package::Docker.new(logger: logger, client: client)
265
+ builder.recipe.apply_input(input)
266
+ begin
267
+ input.input(container)
268
+ return yield(input)
269
+ ensure
270
+ input.cleanup_staging
271
+ input.cleanup_build
272
+ end
273
+ end
274
+
275
+ def write_output!(output)
276
+ package_file = File.expand_path(output.to_s(nil))
277
+ FileUtils.mkdir_p(File.dirname(package_file))
278
+ tmp_package_file = package_file + '.tmp'
279
+ begin
280
+ FileUtils.rm_rf tmp_package_file
281
+ rescue Errno::ENOENT
282
+ end
283
+
284
+ output.output(tmp_package_file)
285
+
286
+ begin
287
+ FileUtils.rm_rf package_file
288
+ rescue Errno::ENOENT
289
+ end
290
+ File.rename tmp_package_file, package_file
291
+
292
+ logger.info("Created package", :path => package_file)
293
+ end
294
+
295
+ def packages
296
+ dir_map = []
297
+ out_map = {}
298
+
299
+ package_map = builder.recipe.packages.map do | package |
300
+ output = output_class.new
301
+ output.instance_variable_set(:@logger,logger)
302
+ package.files.each do | pattern |
303
+ dir_map << [ pattern, output.staging_path ]
304
+ end
305
+ out_map[ output ] = package
306
+ end
307
+
308
+ dir_map = Hash[ dir_map.reverse ]
309
+
310
+ yield dir_map
311
+
312
+ out_map.each do |output, package|
313
+ package.apply_output(output)
314
+ end
315
+
316
+ out_map.each do |output, _|
317
+ write_output!(output)
318
+ end
319
+
320
+ ensure
321
+
322
+ out_map.each do |output, _|
323
+ output.cleanup_staging
324
+ output.cleanup_build
325
+ end
326
+
327
+ end
328
+
329
+
330
+ public
331
+
332
+ def execute
333
+ # force some eager loading
334
+ lint_recipe_file!
335
+ detector
336
+ flavour
337
+ output_class
338
+ lint_output_class!
339
+ builder
340
+ lint_recipe!
341
+ cache
342
+
343
+ image_id
344
+ build_image
345
+
346
+ packages do | dir_map |
347
+
348
+ build! do |container|
349
+ input_package(container) do |input|
350
+ input.split( container, dir_map )
351
+ end
352
+ end
353
+
354
+ end
355
+
356
+ return 0
357
+ rescue Recipe::NotFound => e
358
+ logger.error("Recipe not found", recipe: recipe, exeception: e)
359
+ return 1
360
+ rescue => e
361
+ logger.error(e)
362
+ return 1
363
+ end
364
+
365
+ end
366
+
367
+ class Command
368
+ subcommand 'cook', 'Cooks a package', Cook
369
+ end
370
+ end ; end