fpm-fry 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f7a586c439b3d55bc4f79e6b913e2b263738b0777070e031fc475c7581693e90
-  data.tar.gz: 6dc98b7b8a5dba4bf15d3e6bb11c8c03cbd21a0bface5de566255aa75f966fe1
+  metadata.gz: 125ee923517a62b0c7713f71ede5385e25dfed1fc678f29482fd5a37df8cbb8d
+  data.tar.gz: 48abf7e53d317b5a0be1b2040ee9dd9488e35f7354dca5fd3536276491a74afd
 SHA512:
-  metadata.gz: e560d5ca908265f246ea5017e918b83a41485282c269808a4ee2cb243ffeb51723b7d09aaad27652773e424c29f2012ea61a79b761bbece9f04cdfd96e5b1403
-  data.tar.gz: 194b3169dedc54888baecd8a32e76ce830783219fc9814eef891cd760add2896242872f4e585a70552b27ec02e8ffc07a186c711edee07d2b4084e8518bc0dc3
+  metadata.gz: '097e0ee93510dedb140a39fc4c9b309c54e266c74cc09069bb97c5887526551b8042c2927973bbdf9810e1d470f4c2762ec1c33b03ea6e7baeaaddce6ec7e434'
+  data.tar.gz: 6c9f5bb8522adaf6c84d6656602e398bbd8e52d86667cf88e89c22e8e76ea741582cd491400c64aedde455e64c469081084729d83574664923acce6849d66407

data/lib/fpm/fry/client.rb

@@ -55,13 +55,17 @@ class FPM::Fry::Client
 
   # @return [String] docker server api version
   def server_version
-    @server_version ||= begin
-      res = agent.get(
-        expects: [200],
-        path: '/version'
-      )
-      JSON.parse(res.body)
-    end
+    @server_version ||=
+      begin
+        res = agent.get(
+          expects: [200],
+          path: '/version'
+        )
+        JSON.parse(res.body)
+      rescue Excon::Error => e
+        @logger.error("could not read server version: url: /version, errorr #{e}")
+        raise
+      end
   end
 
   # @return [String] docker cert path from environment
@@ -77,27 +81,34 @@ class FPM::Fry::Client
     tls.any?
   end
 
-
   def url(*path)
-    ['', "v"+server_version['ApiVersion'],*path].join('/')
+    ['', "v"+server_version['ApiVersion'], *path.compact].join('/')
   end
 
   def read(name, resource)
     return to_enum(:read, name, resource) unless block_given?
-    res = if (server_version['ApiVersion'] < "1.20")
-            agent.post(
-              path: url('containers', name, 'copy'),
-              headers: { 'Content-Type' => 'application/json' },
-              body: JSON.generate({'Resource' => resource}),
-              expects: [200,404,500]
-            )
-          else
-            agent.get(
-              path: url('containers', name, 'archive'),
-              headers: { 'Content-Type' => 'application/json' },
-              query: {:path => resource},
-              expects: [200,404,500]
-            )
+    url = nil
+    res = begin
+            if (server_version['ApiVersion'] < "1.20")
+              url = self.url('containers', name, 'copy')
+              agent.post(
+                path: url,
+                headers: { 'Content-Type' => 'application/json' },
+                body: JSON.generate({'Resource' => resource}),
+                expects: [200,404,500]
+              )
+            else
+              url = self.url('containers', name, 'archive')
+              agent.get(
+                path: url,
+                headers: { 'Content-Type' => 'application/json' },
+                query: {:path => resource},
+                expects: [200,404,500]
+              )
+            end
+          rescue Excon::Error => e
+            @logger.error("unexpected response when reading resource: url: #{url}, error: #{e}")
+            raise
           end
     if [404,500].include? res.status
       body_message = Hash[JSON.load(res.body).map{|k,v| ["docker.#{k}",v] }] rescue {'docker.message' => res.body}
@@ -163,9 +174,12 @@ class FPM::Fry::Client
   end
 
   def changes(name)
-    res = agent.get(path: url('containers',name,'changes'))
-    raise res.reason if res.status != 200
+    url = url('containers',name,'changes')
+    res = agent.get(path: url, expects: [200, 204])
     return JSON.parse(res.body)
+  rescue Excon::Error => e
+    @logger.error("could not retrieve changes for: #{name}, url: #{url}, error: #{e}")
+    raise
   end
 
   def pull(image)
@@ -173,20 +187,28 @@ class FPM::Fry::Client
   end
 
   def create(image)
+    url = url('containers','create')
     res = agent.post(
       headers: { 'Content-Type' => 'application/json' },
-      path: url('containers','create'),
-      expects: [201],
+      path: url,
       body: JSON.generate('Image' => image)
     )
     return JSON.parse(res.body)['Id']
+  rescue Excon::Error => e
+    @logger.error("could not create image: #{image}, url: #{url}, error: #{e}")
+    raise
   end
 
   def destroy(container)
+    return unless container
+    url = self.url('containers', container)
     agent.delete(
-      path: url('containers',container),
+      path: url,
       expects: [204]
     )
+  rescue Excon::Error => e
+    @logger.error("could not destroy container: #{container}, url: #{url}, error: #{e}")
+    raise
   end
 
   def agent

data/lib/fpm/fry/command/cook.rb

@@ -4,6 +4,7 @@ module FPM; module Fry
 
     option '--keep', :flag, 'Keep the container after build'
     option '--overwrite', :flag, 'Overwrite package', default: true
+    option '--verbose', :fag, 'Verbose output', default: false
 
     UPDATE_VALUES = ['auto','never','always']
     option '--update',"<#{UPDATE_VALUES.join('|')}>", 'Update image before installing packages ( only apt currently )',attribute_name: 'update', default: 'auto' do |value|
@@ -93,14 +94,16 @@ module FPM; module Fry
     end
 
     def image_id
-      @image_id ||= begin
-        res = client.get(
-          expects: [200],
-          path: client.url("images/#{image}/json")
-        )
-        body = JSON.parse(res.body)
-        body.fetch('id'){ body.fetch('Id') }
-      end
+      @image_id ||=
+        begin
+          url = client.url("images/#{image}/json")
+          res = client.get(expects: [200], path: url)
+          body = JSON.parse(res.body)
+          body.fetch('id'){ body.fetch('Id') }
+        rescue Excon::Error
+          logger.error "could not fetch image json for #{image}, url: #{url}"
+          raise
+        end
     end
     attr_writer :image_id
 
@@ -108,20 +111,32 @@ module FPM; module Fry
       @build_image ||= begin
         sum = Digest::SHA256.hexdigest( image_id + "\0" + cache.cachekey )
         cachetag = "fpm-fry:#{sum[0..30]}"
-        res = client.get(
-          expects: [200,404],
-          path: client.url("images/#{cachetag}/json")
-        )
+        res = begin
+                url = client.url("images/#{cachetag}/json")
+                client.get(
+                  expects: [200,404],
+                  path: url
+                )
+              rescue Excon::Error
+                logger.error "could not fetch image json for #{image}, url: #{url}"
+                raise
+              end
         if res.status == 404
           df = DockerFile::Source.new(builder.variables.merge(image: image_id),cache)
-          client.post(
-            headers: {
-              'Content-Type'=>'application/tar'
-            },
-            expects: [200],
-            path: client.url("build?rm=1&dockerfile=#{DockerFile::NAME}&t=#{cachetag}"),
-            request_block: BlockEnumerator.new(df.tar_io)
-          )
+          begin
+            url = client.url("build?rm=1&dockerfile=#{DockerFile::NAME}&t=#{cachetag}")
+            client.post(
+              headers: {
+                'Content-Type'=>'application/tar'
+              },
+              expects: [200],
+              path: url,
+              request_block: BlockEnumerator.new(df.tar_io)
+            )
+          rescue Excon::Error
+            logger.error "could not build #{image}, url: #{url}"
+            raise
+          end
         else
           # Hack to trigger hints/warnings even when the cache is valid.
           DockerFile::Source.new(builder.variables.merge(image: image_id),cache).send(:file_map)
@@ -129,15 +144,21 @@ module FPM; module Fry
 
         df = DockerFile::Build.new(cachetag, builder.variables.dup,builder.recipe, update: update?)
         parser = BuildOutputParser.new(out)
-        res = client.post(
-          headers: {
-            'Content-Type'=>'application/tar'
-          },
-          expects: [200],
-          path: client.url("build?rm=1&dockerfile=#{DockerFile::NAME}"),
-          request_block: BlockEnumerator.new(df.tar_io),
-          response_block: parser
-        )
+        begin
+          url = client.url("build?rm=1&dockerfile=#{DockerFile::NAME}")
+          res = client.post(
+            headers: {
+              'Content-Type'=>'application/tar'
+            },
+            expects: [200],
+            path: url,
+            request_block: BlockEnumerator.new(df.tar_io),
+            response_block: parser
+          )
+        rescue Excon::Error
+          logger.error "could not build #{image}, url: #{url}"
+          raise
+        end
         if parser.images.none?
           raise "Didn't find a build image in the stream. This usually means that the build script failed."
         end
@@ -175,49 +196,72 @@ module FPM; module Fry
     end
 
     def build!
-      res = client.post(
-         headers: {
-          'Content-Type' => 'application/json'
-         },
-         path: client.url('containers','create'),
-         expects: [201],
-         body: JSON.generate({"Image" => build_image})
-      )
-
-      body = JSON.parse(res.body)
+      body = begin
+              url = client.url('containers','create')
+              res = client.post(
+                headers: {
+                  'Content-Type' => 'application/json'
+                },
+                path: url,
+                expects: [201],
+                body: JSON.generate({"Image" => build_image})
+              )
+              JSON.parse(res.body)
+            rescue Excon::Error
+              logger.error "could not create #{build_image}, url: #{url}"
+              raise
+            end
       container = body['Id']
       begin
-        client.post(
-          headers: {
-            'Content-Type' => 'application/json'
-          },
-          path: client.url('containers',container,'start'),
-          expects: [204],
-          body: JSON.generate({})
-        )
-
-        client.post(
-          path: client.url('containers',container,'attach?stderr=1&stdout=1&stream=1'),
-          body: '',
-          expects: [200],
-          middlewares: [
-            StreamParser.new(out,err),
-            Excon::Middleware::Expects,
-            Excon::Middleware::Instrumentor,
-            Excon::Middleware::Mock
-          ]
-        )
-
-        res = client.post(
-          path: client.url('containers',container,'wait'),
-          expects: [200],
-          body: ''
-        )
-        json = JSON.parse(res.body)
-        if json["StatusCode"] != 0
-          raise Fry::WithData("Build failed", exit_code: json["StatusCode"])
+        begin
+          url = client.url('containers',container,'start')
+          client.post(
+            headers: {
+              'Content-Type' => 'application/json'
+            },
+            path: url,
+            expects: [204],
+            body: JSON.generate({})
+          )
+        rescue Excon::Error
+          logger.error "could not start container #{container}, url: #{url}"
+          raise
         end
-        return yield container
+
+        begin
+          url = client.url('containers',container,'attach?stderr=1&stdout=1&stream=1&logs=1')
+          client.post(
+            path: url,
+            body: '',
+            expects: [200],
+            middlewares: [
+              StreamParser.new(out,err),
+              Excon::Middleware::Expects,
+              Excon::Middleware::Instrumentor,
+              Excon::Middleware::Mock
+            ]
+          )
+        rescue Excon::Error
+          logger.error "could not attach to container #{container}, url: #{url}"
+          raise
+        end
+
+        begin
+          res = client.post(
+            path: client.url('containers',container,'wait'),
+            expects: [200],
+            body: ''
+          )
+          json = JSON.parse(res.body)
+          if json["StatusCode"] != 0
+            raise Fry::WithData("Build failed", exit_code: json["StatusCode"])
+          end
+        rescue Excon::Error
+          logger.error "could not wait successfully for container #{container}, url: #{url}"
+          raise
+        end
+
+        yield container
       ensure
         unless keep?
           client.destroy(container)
@@ -226,7 +270,12 @@ module FPM; module Fry
     end
 
     def input_package(container)
-      input = FPM::Package::Docker.new(logger: logger, client: client)
+      input = FPM::Package::Docker.new(
+        logger: logger,
+        client: client,
+        keep_modified_files: builder.keep_modified_files,
+        verbose: verbose
+      )
       builder.recipe.apply_input(input)
       begin
         input.input(container)
@@ -267,7 +316,7 @@ module FPM; module Fry
       dir_map = []
       out_map = {}
 
-      package_map = builder.recipe.packages.map do | package |
+      builder.recipe.packages.each do | package |
         output = output_class.new
         output.instance_variable_set(:@logger,logger)
         package.files.each do | pattern |

data/lib/fpm/fry/docker_file.rb

@@ -140,6 +140,7 @@ module FPM; module Fry
             if options[:update]
               update = 'apt-get update && '
             end
+            df[:dependencies] << "ENV DEBIAN_FRONTEND=noninteractive"
             df[:dependencies] << "RUN #{update}apt-get install --yes #{Shellwords.join(build_dependencies)}"
           when 'redhat'
             df[:dependencies] << "RUN yum -y install #{Shellwords.join(build_dependencies)}"

data/lib/fpm/fry/plugin/service.rb

@@ -10,11 +10,9 @@ module FPM::Fry::Plugin ; module Service
 
     def render(file)
       _erbout = ""
-      erb = ERB.new(
-        IO.read(File.join(File.dirname(__FILE__),'..','templates',file)),
-        0, "-"
-      )
-      eval(erb.src,nil,File.join(File.dirname(__FILE__),'..','templates',file))
+      path = File.join(File.dirname(__FILE__),'..','templates',file)
+      erb = ERB.new(IO.read(path), trim_mode: "-")
+      eval erb.src, nil, path
       return _erbout
     end
 

data/lib/fpm/fry/recipe/builder.rb

@@ -302,6 +302,12 @@ module FPM::Fry
         PackageBuilder.new(variables, pr, logger: logger, inspector: inspector).instance_eval(&block)
       end
 
+      attr_reader :keep_modified_files
+
+      def keep_modified_files!
+        @keep_modified_files = true
+      end
+
     protected
 
       def source_types

data/lib/fpm/fry/stream_parser.rb

@@ -16,38 +16,56 @@ module FPM; module Fry
         if datum[:response]
           # probably mocked
           if datum[:response][:body]
-            @parser.parse(StringIO.new(datum[:response][:body]))
+            headers = datum[:response][:headers]
+            fake_socket = StringIO.new(datum[:response][:body])
+            parse_response_data(fake_socket, headers)
           end
-          return @stack.response_call(datum)
         else
-          socket = datum[:connection].send(:socket)
-          begin
-            line = socket.readline
-            match = /^HTTP\/\d+\.\d+\s(\d{3})\s/.match(line)
-          end while !match
-          status = match[1].to_i
-
-          datum[:response] = {
-            :body          => '',
-            :headers       => Excon::Headers.new,
-            :status        => status,
-            :remote_ip     => socket.respond_to?(:remote_ip) && socket.remote_ip,
-          }
-          Excon::Response.parse_headers(socket, datum)
+          socket, headers = extract_socket_and_headers(datum)
+          parse_response_data(socket, headers)
+        end
+        @stack.response_call(datum)
+      end
+
+      private
+
+      def extract_socket_and_headers(datum)
+        socket = datum[:connection].send(:socket)
+        begin
+          line = socket.readline
+          match = /^HTTP\/\d+\.\d+\s(\d{3})\s/.match(line)
+        end while !match
+        status = match[1].to_i
+
+        headers = Excon::Headers.new
+        datum[:response] = {
+          :body          => '',
+          :headers       => headers,
+          :status        => status,
+          :remote_ip     => socket.respond_to?(:remote_ip) && socket.remote_ip,
+        }
+        Excon::Response.parse_headers(socket, datum)
+
+        [socket, headers]
+      end
 
+      def parse_response_data(socket, headers)
+        if headers["Transfer-Encoding"] == "chunked"
+          @parser.parse_chunked(socket)
+        else
           @parser.parse(socket)
-          return @stack.response_call(datum)
         end
       end
 
     end
 
-    attr :out, :err
+    attr :out, :err, :streams
 
     def initialize(out, err)
       @out, @err = out, err
       @state = :null
       @left = 0
+      @streams = { 1 => out, 2 => err }
     end
 
     def new(stack)
@@ -55,8 +73,6 @@ module FPM; module Fry
     end
 
     def parse(socket)
-      left  = 0
-      streams = {1 => out, 2 => err}
       loop do
         type = read_exactly(socket,4){|part|
           if part.bytesize == 0
@@ -94,5 +110,16 @@ module FPM; module Fry
       return buf
     end
 
+    def parse_chunked(socket)
+      loop do
+        line = socket.readline
+        chunk_size = line.chomp.to_i(16)
+        break if chunk_size.zero?
+        chunk = socket.read(chunk_size)
+        parse(StringIO.new(chunk))
+        line = socket.readline
+      end
+    end
+
   end
 end ; end

data/lib/fpm/package/docker.rb

@@ -7,23 +7,20 @@ require 'fpm/fry/client'
 # An {FPM::Package} that loads files from a docker container diff.
 class FPM::Package::Docker < FPM::Package
 
+  attr_reader :logger, :client, :keep_modified_files
+
   # @param [Hash] options
   # @option options [Cabin::Channel] :logger Logger
   # @option options [FPM::Fry::Client] :client Docker client
   def initialize( options = {} )
     super()
-    if options[:logger]
-      @logger = options[:logger]
-    end
-    if options[:client]
-      @client = options[:client]
-    end
-    if @logger.nil?
-      @logger = Cabin::Channel.get
-    end
+    @logger = options[:logger] || Cabin::Channel.get
+    @client = options[:client] || FPM::Fry::Client.new(logger: @logger)
+    @keep_modified_files = options[:keep_modified_files]
+    @verbose = options[:verbose]
   end
 
-  # Loads all files from a docker container with the given name to the staging 
+  # Loads all files from a docker container with the given name to the staging
   # path.
   #
   # @param [String] name docker container name
@@ -31,44 +28,47 @@ class FPM::Package::Docker < FPM::Package
     split( name, '**' => staging_path)
   end
 
-  # Loads all files from a docker container into multiple paths defined by map 
+  # Loads all files from a docker container into multiple paths defined by map
   # param.
   #
   # @param [String] name docker container name
-  # @param [Hash<String,String>] map 
+  # @param [Hash<String,String>] map
   def split( name, map )
     changes = changes(name)
-    changes.remove_modified_leaves! do | kind, ml |
+    changes.remove_modified_leaves!(changes_to_remove) do | kind, ml |
       if kind == DELETED
-        @logger.warn("Found a deleted file. You can only create new files in a package",name: ml)
-      else
-        @logger.warn("Found a modified file. You can only create new files in a package",name: ml)
+        logger.warn("Found a deleted file. You can't delete files as part of a package.", name: ml)
+      elsif !keep_modified_files
+        logger.warn("Found a modified file. You can't modify files in a package.", name: ml)
       end
     end
     fmap = {}
     changes.leaves.each do | change |
       map.each do | match, to |
         if File.fnmatch?(match, change)
-          fmap[ change ] = File.join(to, change)
+          fmap[change] = File.join(to, change)
           break
         end
       end
     end
     directories = changes.smallest_superset
     directories.each do |chg|
-      client.copy(name, chg, fmap ,chown: false)
+      client.copy(name, chg, fmap, chown: false)
     end
     return nil
   end
 
-private
-
-  def client
-    @client ||= FPM::Fry::Client.new(logger: @logger)
-  end
+  private
 
   def changes(name)
-    fs = Node.read(client.changes(name))
+    client_changes = client.changes(name)
+    if @verbose
+      names = {MODIFIED => "MOD", CREATED => "ADD", DELETED => "DEL"}
+      client_changes.each do |change|
+        puts [names[change["Kind"]], change["Path"]].join(" ")
+      end
+    end
+    fs = Node.read(client_changes)
     fs.reject!(&method(:ignore?))
     return fs
   end
@@ -95,6 +95,10 @@ private
   CREATED = 1
   DELETED = 2
 
+  def changes_to_remove
+    @keep_modified_files ? [DELETED] : [DELETED, MODIFIED]
+  end
+
   # @api private
   class Node < Struct.new(:children, :kind)
 
@@ -110,7 +114,7 @@ private
       children.none?
     end
 
-    def leaves( prefix = '/', &block )
+    def leaves(prefix = '/', &block)
       return to_enum(:leaves, prefix) unless block
       if leaf?
         yield prefix, false
@@ -126,7 +130,7 @@ private
       children.any?{|_,c| c.leaf? }
     end
 
-    def modified_leaves( prefix = '/', &block )
+    def modified_leaves(prefix = '/', &block)
       return to_enum(:modified_leaves, prefix) unless block
       if leaf?
         if kind != CREATED
@@ -139,11 +143,11 @@ private
       end
     end
 
-    def remove_modified_leaves!( prefix = '/', &block )
+    def remove_modified_leaves!(changes_to_remove, prefix = '/', &block)
       to_remove = {}
       children.each do |name, cld|
-        removed_children = cld.remove_modified_leaves!(File.join(prefix,name), &block)
-        if cld.leaf? and cld.kind != CREATED
+        removed_children = cld.remove_modified_leaves!(changes_to_remove, File.join(prefix,name), &block)
+        if cld.leaf? && changes_to_remove.include?(cld.kind)
           to_remove[name] = [cld.kind, removed_children]
         end
       end
@@ -159,7 +163,7 @@ private
       return false
     end
 
-    def smallest_superset( prefix = '/', &block )
+    def smallest_superset(prefix = '/', &block)
       return to_enum(:smallest_superset, prefix) unless block
       if leaf?
         return
@@ -172,8 +176,8 @@ private
       end
     end
 
-    def reject!( prefix = '/',&block )
-      children.reject! do |name, cld| 
+    def reject!(prefix = '/', &block)
+      children.reject! do |name, cld|
         p = File.join(prefix,name)
         if yield p
           true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fpm-fry
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Maxime Lagresle
@@ -13,7 +13,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-29 00:00:00.000000000 Z
+date: 2022-11-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: excon
@@ -21,14 +21,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.71.0
+        version: '0.71'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.71.0
+        version: '0.71'
 - !ruby/object:Gem::Dependency
   name: fpm
   requirement: !ruby/object:Gem::Requirement
@@ -95,28 +95,28 @@ dependencies:
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.8.23
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.8.23
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: deep-fried package builder
@@ -197,7 +197,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.15
+rubygems_version: 3.3.19
 signing_key:
 specification_version: 4
 summary: FPM Fry