fpm-fry 0.5.1 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 4911dfb40c91dd585e0ccbbad499a4d71eebadd4c00846b982ed100b186c963f
4
- data.tar.gz: 7122bfa7728cb976d3b12010e63be177ccc23d00c360c4830317369f1d1012f1
3
+ metadata.gz: 125ee923517a62b0c7713f71ede5385e25dfed1fc678f29482fd5a37df8cbb8d
4
+ data.tar.gz: 48abf7e53d317b5a0be1b2040ee9dd9488e35f7354dca5fd3536276491a74afd
5
5
  SHA512:
6
- metadata.gz: 9e1ef45f2c7b25b759e3f170b0a3a41d745e686a8a008da43494f6c53166f9695b241f3f2e45dab5fafacfd18a88ae5320e3e7831c4114d5a5e0cbbea08cf10c
7
- data.tar.gz: fbe991bb53e481eab8293c0631079af94795f60dc6381635448fac81aaea0b879fe97e50860688654795a820d2813fab581bc06881223514c289bdd70b9dd42f
6
+ metadata.gz: '097e0ee93510dedb140a39fc4c9b309c54e266c74cc09069bb97c5887526551b8042c2927973bbdf9810e1d470f4c2762ec1c33b03ea6e7baeaaddce6ec7e434'
7
+ data.tar.gz: 6c9f5bb8522adaf6c84d6656602e398bbd8e52d86667cf88e89c22e8e76ea741582cd491400c64aedde455e64c469081084729d83574664923acce6849d66407
@@ -55,13 +55,17 @@ class FPM::Fry::Client
55
55
 
56
56
  # @return [String] docker server api version
57
57
  def server_version
58
- @server_version ||= begin
59
- res = agent.get(
60
- expects: [200],
61
- path: '/version'
62
- )
63
- JSON.parse(res.body)
64
- end
58
+ @server_version ||=
59
+ begin
60
+ res = agent.get(
61
+ expects: [200],
62
+ path: '/version'
63
+ )
64
+ JSON.parse(res.body)
65
+ rescue Excon::Error => e
66
+ @logger.error("could not read server version: url: /version, errorr #{e}")
67
+ raise
68
+ end
65
69
  end
66
70
 
67
71
  # @return [String] docker cert path from environment
@@ -77,27 +81,34 @@ class FPM::Fry::Client
77
81
  tls.any?
78
82
  end
79
83
 
80
-
81
84
  def url(*path)
82
- ['', "v"+server_version['ApiVersion'],*path].join('/')
85
+ ['', "v"+server_version['ApiVersion'], *path.compact].join('/')
83
86
  end
84
87
 
85
88
  def read(name, resource)
86
89
  return to_enum(:read, name, resource) unless block_given?
87
- res = if (server_version['ApiVersion'] < "1.20")
88
- agent.post(
89
- path: url('containers', name, 'copy'),
90
- headers: { 'Content-Type' => 'application/json' },
91
- body: JSON.generate({'Resource' => resource}),
92
- expects: [200,404,500]
93
- )
94
- else
95
- agent.get(
96
- path: url('containers', name, 'archive'),
97
- headers: { 'Content-Type' => 'application/json' },
98
- query: {:path => resource},
99
- expects: [200,404,500]
100
- )
90
+ url = nil
91
+ res = begin
92
+ if (server_version['ApiVersion'] < "1.20")
93
+ url = self.url('containers', name, 'copy')
94
+ agent.post(
95
+ path: url,
96
+ headers: { 'Content-Type' => 'application/json' },
97
+ body: JSON.generate({'Resource' => resource}),
98
+ expects: [200,404,500]
99
+ )
100
+ else
101
+ url = self.url('containers', name, 'archive')
102
+ agent.get(
103
+ path: url,
104
+ headers: { 'Content-Type' => 'application/json' },
105
+ query: {:path => resource},
106
+ expects: [200,404,500]
107
+ )
108
+ end
109
+ rescue Excon::Error => e
110
+ @logger.error("unexpected response when reading resource: url: #{url}, error: #{e}")
111
+ raise
101
112
  end
102
113
  if [404,500].include? res.status
103
114
  body_message = Hash[JSON.load(res.body).map{|k,v| ["docker.#{k}",v] }] rescue {'docker.message' => res.body}
@@ -163,9 +174,12 @@ class FPM::Fry::Client
163
174
  end
164
175
 
165
176
  def changes(name)
166
- res = agent.get(path: url('containers',name,'changes'))
167
- raise res.reason if res.status != 200
177
+ url = url('containers',name,'changes')
178
+ res = agent.get(path: url, expects: [200, 204])
168
179
  return JSON.parse(res.body)
180
+ rescue Excon::Error => e
181
+ @logger.error("could not retrieve changes for: #{name}, url: #{url}, error: #{e}")
182
+ raise
169
183
  end
170
184
 
171
185
  def pull(image)
@@ -173,20 +187,28 @@ class FPM::Fry::Client
173
187
  end
174
188
 
175
189
  def create(image)
190
+ url = url('containers','create')
176
191
  res = agent.post(
177
192
  headers: { 'Content-Type' => 'application/json' },
178
- path: url('containers','create'),
179
- expects: [201],
193
+ path: url,
180
194
  body: JSON.generate('Image' => image)
181
195
  )
182
196
  return JSON.parse(res.body)['Id']
197
+ rescue Excon::Error => e
198
+ @logger.error("could not create image: #{image}, url: #{url}, error: #{e}")
199
+ raise
183
200
  end
184
201
 
185
202
  def destroy(container)
203
+ return unless container
204
+ url = self.url('containers', container)
186
205
  agent.delete(
187
- path: url('containers',container),
206
+ path: url,
188
207
  expects: [204]
189
208
  )
209
+ rescue Excon::Error => e
210
+ @logger.error("could not destroy container: #{container}, url: #{url}, error: #{e}")
211
+ raise
190
212
  end
191
213
 
192
214
  def agent
@@ -4,6 +4,7 @@ module FPM; module Fry
4
4
 
5
5
  option '--keep', :flag, 'Keep the container after build'
6
6
  option '--overwrite', :flag, 'Overwrite package', default: true
7
+ option '--verbose', :fag, 'Verbose output', default: false
7
8
 
8
9
  UPDATE_VALUES = ['auto','never','always']
9
10
  option '--update',"<#{UPDATE_VALUES.join('|')}>", 'Update image before installing packages ( only apt currently )',attribute_name: 'update', default: 'auto' do |value|
@@ -93,14 +94,16 @@ module FPM; module Fry
93
94
  end
94
95
 
95
96
  def image_id
96
- @image_id ||= begin
97
- res = client.get(
98
- expects: [200],
99
- path: client.url("images/#{image}/json")
100
- )
101
- body = JSON.parse(res.body)
102
- body.fetch('id'){ body.fetch('Id') }
103
- end
97
+ @image_id ||=
98
+ begin
99
+ url = client.url("images/#{image}/json")
100
+ res = client.get(expects: [200], path: url)
101
+ body = JSON.parse(res.body)
102
+ body.fetch('id'){ body.fetch('Id') }
103
+ rescue Excon::Error
104
+ logger.error "could not fetch image json for #{image}, url: #{url}"
105
+ raise
106
+ end
104
107
  end
105
108
  attr_writer :image_id
106
109
 
@@ -108,20 +111,32 @@ module FPM; module Fry
108
111
  @build_image ||= begin
109
112
  sum = Digest::SHA256.hexdigest( image_id + "\0" + cache.cachekey )
110
113
  cachetag = "fpm-fry:#{sum[0..30]}"
111
- res = client.get(
112
- expects: [200,404],
113
- path: client.url("images/#{cachetag}/json")
114
- )
114
+ res = begin
115
+ url = client.url("images/#{cachetag}/json")
116
+ client.get(
117
+ expects: [200,404],
118
+ path: url
119
+ )
120
+ rescue Excon::Error
121
+ logger.error "could not fetch image json for #{image}, url: #{url}"
122
+ raise
123
+ end
115
124
  if res.status == 404
116
125
  df = DockerFile::Source.new(builder.variables.merge(image: image_id),cache)
117
- client.post(
118
- headers: {
119
- 'Content-Type'=>'application/tar'
120
- },
121
- expects: [200],
122
- path: client.url("build?rm=1&dockerfile=#{DockerFile::NAME}&t=#{cachetag}"),
123
- request_block: BlockEnumerator.new(df.tar_io)
124
- )
126
+ begin
127
+ url = client.url("build?rm=1&dockerfile=#{DockerFile::NAME}&t=#{cachetag}")
128
+ client.post(
129
+ headers: {
130
+ 'Content-Type'=>'application/tar'
131
+ },
132
+ expects: [200],
133
+ path: url,
134
+ request_block: BlockEnumerator.new(df.tar_io)
135
+ )
136
+ rescue Excon::Error
137
+ logger.error "could not build #{image}, url: #{url}"
138
+ raise
139
+ end
125
140
  else
126
141
  # Hack to trigger hints/warnings even when the cache is valid.
127
142
  DockerFile::Source.new(builder.variables.merge(image: image_id),cache).send(:file_map)
@@ -129,15 +144,21 @@ module FPM; module Fry
129
144
 
130
145
  df = DockerFile::Build.new(cachetag, builder.variables.dup,builder.recipe, update: update?)
131
146
  parser = BuildOutputParser.new(out)
132
- res = client.post(
133
- headers: {
134
- 'Content-Type'=>'application/tar'
135
- },
136
- expects: [200],
137
- path: client.url("build?rm=1&dockerfile=#{DockerFile::NAME}"),
138
- request_block: BlockEnumerator.new(df.tar_io),
139
- response_block: parser
140
- )
147
+ begin
148
+ url = client.url("build?rm=1&dockerfile=#{DockerFile::NAME}")
149
+ res = client.post(
150
+ headers: {
151
+ 'Content-Type'=>'application/tar'
152
+ },
153
+ expects: [200],
154
+ path: url,
155
+ request_block: BlockEnumerator.new(df.tar_io),
156
+ response_block: parser
157
+ )
158
+ rescue Excon::Error
159
+ logger.error "could not build #{image}, url: #{url}"
160
+ raise
161
+ end
141
162
  if parser.images.none?
142
163
  raise "Didn't find a build image in the stream. This usually means that the build script failed."
143
164
  end
@@ -175,49 +196,72 @@ module FPM; module Fry
175
196
  end
176
197
 
177
198
  def build!
178
- res = client.post(
179
- headers: {
180
- 'Content-Type' => 'application/json'
181
- },
182
- path: client.url('containers','create'),
183
- expects: [201],
184
- body: JSON.generate({"Image" => build_image})
185
- )
186
-
187
- body = JSON.parse(res.body)
199
+ body = begin
200
+ url = client.url('containers','create')
201
+ res = client.post(
202
+ headers: {
203
+ 'Content-Type' => 'application/json'
204
+ },
205
+ path: url,
206
+ expects: [201],
207
+ body: JSON.generate({"Image" => build_image})
208
+ )
209
+ JSON.parse(res.body)
210
+ rescue Excon::Error
211
+ logger.error "could not create #{build_image}, url: #{url}"
212
+ raise
213
+ end
188
214
  container = body['Id']
189
215
  begin
190
- client.post(
191
- headers: {
192
- 'Content-Type' => 'application/json'
193
- },
194
- path: client.url('containers',container,'start'),
195
- expects: [204],
196
- body: JSON.generate({})
197
- )
198
-
199
- client.post(
200
- path: client.url('containers',container,'attach?stderr=1&stdout=1&stream=1&logs=1'),
201
- body: '',
202
- expects: [200],
203
- middlewares: [
204
- StreamParser.new(out,err),
205
- Excon::Middleware::Expects,
206
- Excon::Middleware::Instrumentor,
207
- Excon::Middleware::Mock
208
- ]
209
- )
210
-
211
- res = client.post(
212
- path: client.url('containers',container,'wait'),
213
- expects: [200],
214
- body: ''
215
- )
216
- json = JSON.parse(res.body)
217
- if json["StatusCode"] != 0
218
- raise Fry::WithData("Build failed", exit_code: json["StatusCode"])
216
+ begin
217
+ url = client.url('containers',container,'start')
218
+ client.post(
219
+ headers: {
220
+ 'Content-Type' => 'application/json'
221
+ },
222
+ path: url,
223
+ expects: [204],
224
+ body: JSON.generate({})
225
+ )
226
+ rescue Excon::Error
227
+ logger.error "could not start container #{container}, url: #{url}"
228
+ raise
219
229
  end
220
- return yield container
230
+
231
+ begin
232
+ url = client.url('containers',container,'attach?stderr=1&stdout=1&stream=1&logs=1')
233
+ client.post(
234
+ path: url,
235
+ body: '',
236
+ expects: [200],
237
+ middlewares: [
238
+ StreamParser.new(out,err),
239
+ Excon::Middleware::Expects,
240
+ Excon::Middleware::Instrumentor,
241
+ Excon::Middleware::Mock
242
+ ]
243
+ )
244
+ rescue Excon::Error
245
+ logger.error "could not attach to container #{container}, url: #{url}"
246
+ raise
247
+ end
248
+
249
+ begin
250
+ res = client.post(
251
+ path: client.url('containers',container,'wait'),
252
+ expects: [200],
253
+ body: ''
254
+ )
255
+ json = JSON.parse(res.body)
256
+ if json["StatusCode"] != 0
257
+ raise Fry::WithData("Build failed", exit_code: json["StatusCode"])
258
+ end
259
+ rescue Excon::Error
260
+ logger.error "could not wait successfully for container #{container}, url: #{url}"
261
+ raise
262
+ end
263
+
264
+ yield container
221
265
  ensure
222
266
  unless keep?
223
267
  client.destroy(container)
@@ -226,7 +270,12 @@ module FPM; module Fry
226
270
  end
227
271
 
228
272
  def input_package(container)
229
- input = FPM::Package::Docker.new(logger: logger, client: client)
273
+ input = FPM::Package::Docker.new(
274
+ logger: logger,
275
+ client: client,
276
+ keep_modified_files: builder.keep_modified_files,
277
+ verbose: verbose
278
+ )
230
279
  builder.recipe.apply_input(input)
231
280
  begin
232
281
  input.input(container)
@@ -267,7 +316,7 @@ module FPM; module Fry
267
316
  dir_map = []
268
317
  out_map = {}
269
318
 
270
- package_map = builder.recipe.packages.map do | package |
319
+ builder.recipe.packages.each do | package |
271
320
  output = output_class.new
272
321
  output.instance_variable_set(:@logger,logger)
273
322
  package.files.each do | pattern |
@@ -140,6 +140,7 @@ module FPM; module Fry
140
140
  if options[:update]
141
141
  update = 'apt-get update && '
142
142
  end
143
+ df[:dependencies] << "ENV DEBIAN_FRONTEND=noninteractive"
143
144
  df[:dependencies] << "RUN #{update}apt-get install --yes #{Shellwords.join(build_dependencies)}"
144
145
  when 'redhat'
145
146
  df[:dependencies] << "RUN yum -y install #{Shellwords.join(build_dependencies)}"
@@ -302,6 +302,12 @@ module FPM::Fry
302
302
  PackageBuilder.new(variables, pr, logger: logger, inspector: inspector).instance_eval(&block)
303
303
  end
304
304
 
305
+ attr_reader :keep_modified_files
306
+
307
+ def keep_modified_files!
308
+ @keep_modified_files = true
309
+ end
310
+
305
311
  protected
306
312
 
307
313
  def source_types
@@ -7,23 +7,20 @@ require 'fpm/fry/client'
7
7
  # An {FPM::Package} that loads files from a docker container diff.
8
8
  class FPM::Package::Docker < FPM::Package
9
9
 
10
+ attr_reader :logger, :client, :keep_modified_files
11
+
10
12
  # @param [Hash] options
11
13
  # @option options [Cabin::Channel] :logger Logger
12
14
  # @option options [FPM::Fry::Client] :client Docker client
13
15
  def initialize( options = {} )
14
16
  super()
15
- if options[:logger]
16
- @logger = options[:logger]
17
- end
18
- if options[:client]
19
- @client = options[:client]
20
- end
21
- if @logger.nil?
22
- @logger = Cabin::Channel.get
23
- end
17
+ @logger = options[:logger] || Cabin::Channel.get
18
+ @client = options[:client] || FPM::Fry::Client.new(logger: @logger)
19
+ @keep_modified_files = options[:keep_modified_files]
20
+ @verbose = options[:verbose]
24
21
  end
25
22
 
26
- # Loads all files from a docker container with the given name to the staging
23
+ # Loads all files from a docker container with the given name to the staging
27
24
  # path.
28
25
  #
29
26
  # @param [String] name docker container name
@@ -31,44 +28,47 @@ class FPM::Package::Docker < FPM::Package
31
28
  split( name, '**' => staging_path)
32
29
  end
33
30
 
34
- # Loads all files from a docker container into multiple paths defined by map
31
+ # Loads all files from a docker container into multiple paths defined by map
35
32
  # param.
36
33
  #
37
34
  # @param [String] name docker container name
38
- # @param [Hash<String,String>] map
35
+ # @param [Hash<String,String>] map
39
36
  def split( name, map )
40
37
  changes = changes(name)
41
- changes.remove_modified_leaves! do | kind, ml |
38
+ changes.remove_modified_leaves!(changes_to_remove) do | kind, ml |
42
39
  if kind == DELETED
43
- @logger.warn("Found a deleted file. You can only create new files in a package",name: ml)
44
- else
45
- @logger.warn("Found a modified file. You can only create new files in a package",name: ml)
40
+ logger.warn("Found a deleted file. You can't delete files as part of a package.", name: ml)
41
+ elsif !keep_modified_files
42
+ logger.warn("Found a modified file. You can't modify files in a package.", name: ml)
46
43
  end
47
44
  end
48
45
  fmap = {}
49
46
  changes.leaves.each do | change |
50
47
  map.each do | match, to |
51
48
  if File.fnmatch?(match, change)
52
- fmap[ change ] = File.join(to, change)
49
+ fmap[change] = File.join(to, change)
53
50
  break
54
51
  end
55
52
  end
56
53
  end
57
54
  directories = changes.smallest_superset
58
55
  directories.each do |chg|
59
- client.copy(name, chg, fmap ,chown: false)
56
+ client.copy(name, chg, fmap, chown: false)
60
57
  end
61
58
  return nil
62
59
  end
63
60
 
64
- private
65
-
66
- def client
67
- @client ||= FPM::Fry::Client.new(logger: @logger)
68
- end
61
+ private
69
62
 
70
63
  def changes(name)
71
- fs = Node.read(client.changes(name))
64
+ client_changes = client.changes(name)
65
+ if @verbose
66
+ names = {MODIFIED => "MOD", CREATED => "ADD", DELETED => "DEL"}
67
+ client_changes.each do |change|
68
+ puts [names[change["Kind"]], change["Path"]].join(" ")
69
+ end
70
+ end
71
+ fs = Node.read(client_changes)
72
72
  fs.reject!(&method(:ignore?))
73
73
  return fs
74
74
  end
@@ -95,6 +95,10 @@ private
95
95
  CREATED = 1
96
96
  DELETED = 2
97
97
 
98
+ def changes_to_remove
99
+ @keep_modified_files ? [DELETED] : [DELETED, MODIFIED]
100
+ end
101
+
98
102
  # @api private
99
103
  class Node < Struct.new(:children, :kind)
100
104
 
@@ -110,7 +114,7 @@ private
110
114
  children.none?
111
115
  end
112
116
 
113
- def leaves( prefix = '/', &block )
117
+ def leaves(prefix = '/', &block)
114
118
  return to_enum(:leaves, prefix) unless block
115
119
  if leaf?
116
120
  yield prefix, false
@@ -126,7 +130,7 @@ private
126
130
  children.any?{|_,c| c.leaf? }
127
131
  end
128
132
 
129
- def modified_leaves( prefix = '/', &block )
133
+ def modified_leaves(prefix = '/', &block)
130
134
  return to_enum(:modified_leaves, prefix) unless block
131
135
  if leaf?
132
136
  if kind != CREATED
@@ -139,11 +143,11 @@ private
139
143
  end
140
144
  end
141
145
 
142
- def remove_modified_leaves!( prefix = '/', &block )
146
+ def remove_modified_leaves!(changes_to_remove, prefix = '/', &block)
143
147
  to_remove = {}
144
148
  children.each do |name, cld|
145
- removed_children = cld.remove_modified_leaves!(File.join(prefix,name), &block)
146
- if cld.leaf? and cld.kind != CREATED
149
+ removed_children = cld.remove_modified_leaves!(changes_to_remove, File.join(prefix,name), &block)
150
+ if cld.leaf? && changes_to_remove.include?(cld.kind)
147
151
  to_remove[name] = [cld.kind, removed_children]
148
152
  end
149
153
  end
@@ -159,7 +163,7 @@ private
159
163
  return false
160
164
  end
161
165
 
162
- def smallest_superset( prefix = '/', &block )
166
+ def smallest_superset(prefix = '/', &block)
163
167
  return to_enum(:smallest_superset, prefix) unless block
164
168
  if leaf?
165
169
  return
@@ -172,8 +176,8 @@ private
172
176
  end
173
177
  end
174
178
 
175
- def reject!( prefix = '/',&block )
176
- children.reject! do |name, cld|
179
+ def reject!(prefix = '/', &block)
180
+ children.reject! do |name, cld|
177
181
  p = File.join(prefix,name)
178
182
  if yield p
179
183
  true
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fpm-fry
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.1
4
+ version: 0.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maxime Lagresle
@@ -13,7 +13,7 @@ authors:
13
13
  autorequire:
14
14
  bindir: bin
15
15
  cert_chain: []
16
- date: 2022-07-29 00:00:00.000000000 Z
16
+ date: 2022-11-27 00:00:00.000000000 Z
17
17
  dependencies:
18
18
  - !ruby/object:Gem::Dependency
19
19
  name: excon
@@ -21,14 +21,14 @@ dependencies:
21
21
  requirements:
22
22
  - - "~>"
23
23
  - !ruby/object:Gem::Version
24
- version: 0.71.0
24
+ version: '0.71'
25
25
  type: :runtime
26
26
  prerelease: false
27
27
  version_requirements: !ruby/object:Gem::Requirement
28
28
  requirements:
29
29
  - - "~>"
30
30
  - !ruby/object:Gem::Version
31
- version: 0.71.0
31
+ version: '0.71'
32
32
  - !ruby/object:Gem::Dependency
33
33
  name: fpm
34
34
  requirement: !ruby/object:Gem::Requirement
@@ -197,7 +197,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
197
197
  - !ruby/object:Gem::Version
198
198
  version: '0'
199
199
  requirements: []
200
- rubygems_version: 3.3.15
200
+ rubygems_version: 3.3.19
201
201
  signing_key:
202
202
  specification_version: 4
203
203
  summary: FPM Fry