buildizer 0.0.6 → 0.0.7

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZmJhNmZhMzlhYzZhZDRjZWYwNTFkZDJhYzU4YjNmMTk3ZjA5MWVhOQ==
+    NTgxMGMxNTEzOTg5MDhjMmExZGUwOWM3M2JjMmM0NjdhYjNkODBiOA==
   data.tar.gz: !binary |-
-    MzZjZTE0MzQ3ODI2YzBkOTc5NzRjOGZhYjdhM2YyYjhhOWFjOWI4Yw==
+    MjRhZjU1NzlmOGM3ZjMyMzEzMDk5NDY5ZmQ0ZGJhZGY1ZGI5YjdhMQ==
 SHA512:
   metadata.gz: !binary |-
-    ZThhYTdlZjNmNTdmZDdmNzRhMTVjOGY1M2U3YjY1NzcxZTRhZTMwZDZiNTc2
-    YzZmZDY1NjhlYTI2ZWIzZmEzNTZhZDQ4NDE2M2ZiZmZmYzBkMTJkOGQwNzBj
-    YzA3MTYzMGNhY2I0YzgwOGU2Njc4MWE1OTVhZmE5M2RlYzk1OGY=
+    Nzk3ZjIyMzVlMmNhOTczMDk5YjQ3NjA3NTIzNjVhZGE4MTY4NjQzMWY4ZjQ5
+    NDc2OWIxZmYyYTE3NzgxMTE4N2ZkYjNmZDQxNzVkMzFjNzc1MWNjMWNmODI0
+    ZmI2MzZjYzhiYzY4MmFkMjExMTM3MDJkMDA3M2MzYTNmYmU3ZDk=
   data.tar.gz: !binary |-
-    OTIxMGYzZDI0N2JmYmQyZDcwYjdmM2QyNGRjMjg0NzE2NTUzNTJkMDFmY2I2
-    MGQ0NDhlNjlkMWFhNmFhYzMyOTVmZDkxZGUzNTk5NWNiOThkNzA2ZjExNzc2
-    NjZiN2NjOGVhY2U2YjE0MDg3NDAwOWZjYzM0YzAwMTI1NzU5Mzc=
+    YTljMjYyZmJiZDdmY2E3MDRhNTVjNjY1ZTJmMWQ3NjUyYWU5YTAxNTU4OTE3
+    YjRlOTJhOTA4YTc5NjExZjk0ZWQxOTE3ZmIzMThhYjU2YTc2YmQ4OGRkMmVm
+    OTNlNWFmZDNmMDBmYWQ2OTE2YWE3ZWQwZGRjYzhjMDQyNjJkNDg=

data/.gitignore

@@ -3,3 +3,4 @@
 /buildizer*.gem
 *.swp
 .tags
+/.bundle

data/Gemfile.lock

@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/flant/shellfold.git
-  revision: 51d8ff898ae1afbfb6463ed2e1ed25571cab347c
+  revision: 7429a46664b45ab246ba7a88eac24f614d1945a7
   specs:
     shellfold (0.0.3)
       mixlib-shellout (>= 2.2.6, < 3.0)
@@ -8,7 +8,7 @@ GIT
 PATH
   remote: .
   specs:
-    buildizer (0.0.6)
+    buildizer (0.0.7)
       net_status (>= 0.0.1, < 1.0)
       package_cloud (>= 0.2, < 1.0)
       shellfold (>= 0.0.1, < 1.0)
@@ -75,7 +75,7 @@ GEM
       rspec-core (~> 3.4.0)
       rspec-expectations (~> 3.4.0)
       rspec-mocks (~> 3.4.0)
-    rspec-core (3.4.3)
+    rspec-core (3.4.4)
       rspec-support (~> 3.4.0)
     rspec-expectations (3.4.0)
       diff-lcs (>= 1.2.0, < 2.0)
@@ -97,7 +97,7 @@ GEM
       typhoeus (~> 0.6, >= 0.6.8)
     typhoeus (0.8.0)
       ethon (>= 0.8.0)
-    websocket (1.2.2)
+    websocket (1.2.3)
 
 PLATFORMS
   ruby

data/lib/buildizer.rb

@@ -1,5 +1,6 @@
 require 'thor'
 require 'yaml'
+require 'json'
 require 'net_status'
 require 'pathname'
 require 'shellfold'
@@ -11,13 +12,16 @@ require 'buildizer/docker'
 require 'buildizer/cli'
 require 'buildizer/packager'
 require 'buildizer/target'
+require 'buildizer/target/package_name_mod'
 require 'buildizer/target/base'
 require 'buildizer/target/fpm'
 require 'buildizer/target/native'
+require 'buildizer/target/patch'
 require 'buildizer/builder'
 require 'buildizer/builder/base'
 require 'buildizer/builder/fpm'
 require 'buildizer/builder/native'
+require 'buildizer/builder/patch'
 require 'buildizer/image'
 require 'buildizer/image/base'
 require 'buildizer/image/centos'
@@ -26,3 +30,4 @@ require 'buildizer/image/centos7'
 require 'buildizer/image/ubuntu'
 require 'buildizer/image/ubuntu1204'
 require 'buildizer/image/ubuntu1404'
+require 'buildizer/image/ubuntu1604'

data/lib/buildizer/builder/base.rb

@@ -1,23 +1,19 @@
 module Buildizer
   module Builder
     class Base
+      using Refine
+
       attr_reader :packager
-      attr_reader :build_path
+      attr_reader :work_path
       attr_reader :docker
 
       def initialize(packager)
         @packager = packager
 
-        @build_path = Pathname.new(ENV['PREFIX'] || packager.package_path.join('build')).expand_path
-        raise Error, message: "bad build prefix: '#{build_path}' is a file" if build_path.file?
-        build_path.mkdir rescue nil
+        @work_path = packager.work_path.join('builder').expand_path
+        work_path.mkpath
 
-        @docker = Docker.new(self,
-          username: packager.docker_username,
-          password: packager.docker_password,
-          email: packager.docker_email,
-          server: packager.docker_server,
-        )
+        @docker = Docker.new(self, cache: packager.docker_cache)
       end
 
       def build_type
@@ -28,24 +24,28 @@ module Buildizer
         raise
       end
 
-      def prepare_image_instructions(target)
-      end
-
       def build_instructions(target)
       end
 
-      def build_dep
+      def build_dep(target)
       end
 
       def new_target(target_name)
-        os_name, os_version, target_package_name, target_package_version = target_name.split('-', 4)
+        os_name, os_version, target_tag = target_name.split('/', 3)
 
         image = docker.new_image(os_name, os_version)
 
-        params = merge_os_params(image.os_name)
-        params = merge_os_version_params(image.os_name, image.os_version, into: params)
-        params = merge_base_target_params(target_name, target_package_name, target_package_version,
-                                          into: params) if target_package_name
+        params = initial_target_params
+        packager.buildizer_conf.each do |match_key, match_params|
+          match_os_name, match_os_version, match_target_tag = match_key.to_s.split('/', 3)
+          if image.os_name.match_glob?(match_os_name) and
+            ( match_os_version.nil? or image.os_version.match_glob?(match_os_version) ) and
+              ( match_target_tag.nil? or (not target_tag.nil? and
+                                          target_tag.match_glob?(match_target_tag)) )
+            params = merge_params(into: params, params: match_params)
+          end
+        end
+
         check_params! params
 
         target_klass.new(self, image, name: target_name, **params).tap do |target|
@@ -65,6 +65,7 @@ module Buildizer
           params[:prepare] = packager.prepare
           params[:build_dep] = packager.build_dep
           params[:before_build] = packager.before_build
+          params[:maintainer] = packager.maintainer
         end
       end
 
@@ -77,96 +78,138 @@ module Buildizer
 
       def do_merge_params(into, params)
         {}.tap do |res|
-          res[:package_name] = into[:package_name] || params['package_name']
-          res[:package_version] = into[:package_version] || params['package_version']
+          res[:package_name] = params['package_name'] || into[:package_name]
+          res[:package_version] = params['package_version'] || into[:package_version]
           res[:package_cloud] = into[:package_cloud]
           res[:prepare] = into[:prepare] + Array(params['prepare'])
           res[:build_dep] = into[:build_dep] | Array(params['build_dep']).to_set
           res[:before_build] = into[:before_build] + Array(params['before_build'])
+          res[:maintainer] = params['maintainer'] || into[:maintainer]
         end
       end
 
-      def merge_os_params(os_name, into: nil, &blk)
-        merge_params(into: into, params: packager.os_params(os_name), &blk)
+      def check_params!(params)
+        _required_params! :package_name, params
       end
 
-      def merge_os_version_params(os_name, os_version, into: nil, &blk)
-        merge_params(into: into,
-                     params: packager.os_params([os_name, os_version].join('-')), &blk)
+      def _required_params!(required_params, params)
+        Array(required_params).each do |param|
+          unless params[param] and not params[param].to_s.empty?
+            raise Error, error: :input_error, message: "#{param} is not defined"
+          end
+        end
       end
 
-      def merge_base_target_params(target, target_package_name, target_package_version,
-                                   into: nil, &blk)
-        merge_params(into: into,
-                     params: {'package_name' => target_package_name,
-                              'package_version' => target_package_version}, &blk)
+      def build_jobs
+        File.open('/proc/cpuinfo').readlines.grep(/processor/).size
       end
 
-      def check_params!(params)
-        [:package_name, :package_version, :package_cloud].each do |param|
-          raise(Error,
-                error: :input_error,
-                message: "#{param} is not defined") unless params[param] and not params[param].empty?
+      def verify
+        targets.tap do |res|
+          unless res.any?
+            raise Error, error: :input_error, message: "target is not defined"
+          end
         end
       end
 
       def prepare
-        return unless packager.enabled?
+        docker.with_cache do
+          packager.before_prepare
+                  .each {|cmd| packager.command! cmd, desc: "Before prepare command: #{cmd}"}
 
-        docker.login!
-
-        begin
-          packager.before_prepare.each {|cmd| packager.command! cmd, desc: "Before prepare command: #{cmd}"}
           targets.each {|target| prepare_target_image(target)}
-          packager.after_prepare.each {|cmd| packager.command! cmd, desc: "After prepare command: #{cmd}"}
-        ensure
-          docker.logout!
-        end
+
+          packager.after_prepare
+                  .each {|cmd| packager.command! cmd, desc: "After prepare command: #{cmd}"}
+        end # with_cache
       end
 
       def prepare_target_image(target)
-        docker.image_build_path(target.image).mkpath
-
-        (Array(prepare_image_instructions(target)) + target.prepare).each do |cmd|
-          target.image.instruction(:RUN, "bash -lec \"#{cmd}\"")
-        end
-        target.image.build_dep(Array(build_dep).to_set + target.build_dep)
-        docker.build_image! target.image
+        target.prepare.each {|cmd| target.image.instruction(:RUN, "bash -lec \"#{cmd}\"")}
+        target.image.build_dep(Array(build_dep(target)).to_set + target.build_dep)
+        docker.build_image! target
       end
 
       def build
-        return unless packager.enabled?
         targets.each {|target| build_target(target)}
       end
 
-      def build_target(target)
-        docker.image_runtime_build_path(target.image).mkpath
+      def prepare_package_source_instructions(target)
+        ["cp -r #{docker.container_package_mount_path} #{target.container_package_path}",
+         "rm -rf #{target.container_package_path.join('.git')}",
+         "cd #{target.container_package_path.dirname}",
+         ["tar -zcvf #{target.container_package_archive_path} ",
+          "#{target.container_package_path.basename}"].join,
+         "ln -fs #{target.container_package_path} #{docker.container_package_path}",
+         "ln -fs #{target.container_package_archive_path} #{docker.container_package_archive_path}"]
+      end
 
+      def build_target(target)
         cmd = [
-          "cd /package",
+          *Array(prepare_package_source_instructions(target)),
+          "rm -rf #{docker.container_build_path.join('*')}",
+          "cd #{docker.container_package_path}",
           *target.before_build,
           *Array(build_instructions(target)),
         ]
 
-        docker.run! target.image, cmd: cmd
+        docker.run_in_image!(target: target, cmd: cmd,
+                             desc: "Run build in docker image '#{target.image.name}'")
       end
 
       def deploy
-        return unless packager.enabled?
-        targets.each {|target| deploy_target(target)}
+        if packager.package_version_tag_required_for_deploy? and
+           not packager.package_version_tag
+          puts "package_version_tag (env TRAVIS_TAG or CI_BUILD_TAG) required: ignoring deploy"
+          return
+        elsif packager.package_cloud.empty?
+          warn "No package cloud settings " +
+               "(PACKAGECLOUD, PACKAGECLOUD_TOKEN, PACKAGECLOUD_TOKEN_<ORG>) [WARN]"
+          return
+        end
+
+        packager.package_cloud_org.each do |org, token|
+          unless token
+            warn "No packagecloud token defined for org '#{org}' " +
+                 "(PACKAGECLOUD_TOKEN or PACKAGECLOUD_TOKEN_#{org.upcase}) [WARN]"
+          end
+        end
+
+        targets.map do |target|
+          target.tap do
+            if packager.package_version_tag_required_for_deploy? and
+               packager.package_version_tag != target.package_version_tag
+              raise(Error, error: :logical_error,
+                           message: "#{target.package_version_tag_param_name} and "+
+                                    "package_version_tag (env TRAVIS_TAG or CI_BUILD_TAG) " +
+                                    "should be the same for target '#{target.name}'")
+            end
+          end
+        end.each {|target| deploy_target(target)}
       end
 
       def deploy_target(target)
-        cmd = Dir[docker.image_runtime_build_path(target.image)
-                        .join("*.#{target.image.fpm_output_type}")]
-                  .map {|p| Pathname.new(p)}
-                  .map {|p| ["package_cloud yank #{target.package_cloud_path} #{p.basename}",
-                             "package_cloud push #{target.package_cloud_path} #{p}",
-                             p.basename]}
-                  .each {|yank, push, package|
-                    packager.command yank, desc: "Package cloud yank package '#{package}'"
-                    packager.command! push, desc: "Package cloud push package '#{package}'"
-                  }
+        cmd = Dir[target.image_build_path.join("*.#{target.image.fpm_output_type}")]
+                .map {|p| Pathname.new(p)}.map {|package_path|
+                  package = package_path.basename
+                  target.package_cloud.map do |desc|
+                    desc.merge(
+                      package: package,
+                      yank: "package_cloud yank #{desc[:package_path]} #{package}",
+                      push: "package_cloud push #{desc[:package_path]} #{package_path}",
+                    )
+                  end
+                }.flatten.each {|desc|
+                  packager.command desc[:yank],
+                    desc: ["Package cloud yank package '#{desc[:package]}'",
+                           " of target '#{target.name}'"].join,
+                    environment: {'PACKAGECLOUD_TOKEN' => desc[:token]}
+
+                  packager.command desc[:push],
+                    desc: ["Package cloud push package '#{desc[:package]}'",
+                           " of target '#{target.name}'"].join,
+                    environment: {'PACKAGECLOUD_TOKEN' =>desc[:token]}
+                }
       end
     end # Base
   end # Builder

data/lib/buildizer/builder/fpm.rb

@@ -15,60 +15,29 @@ module Buildizer
 
       def initial_target_params
         super.tap do |params|
-          raise(Error,
-                error: :input_error,
-                message: [
-                  "explicit definition of package_version in #{build_type} ",
-                  "build type is forbidden ",
-                  "(use TRAVIS_TAG, CI_BUILD_TAG env variables)",
-                ].join) if params[:package_version]
-          params[:package_version] = packager.package_version_tag
           params[:fpm_script] = Array(packager.buildizer_conf['fpm_script'])
           params[:fpm_config_files] = packager.buildizer_conf['fpm_config_files'].to_h
           params[:fpm_files] = packager.buildizer_conf['fpm_files'].to_h
-        end
-      end
-
-      def cannot_redefine_package_params!(params, redefine_for: nil)
-        [:package_name, :package_version].each do |param|
-          raise(Error,
-            error: :input_error,
-            message: [
-              "cannot redefine #{param}",
-              redefine_for ? "for #{redefine_for}" : nil,
-              "in #{build_type} build_type",
-            ].compact.join(' ')
-          ) if params.key? param.to_s
+          params[:fpm_conflicts] = Array(packager.buildizer_conf['fpm_conflicts'])
+          params[:fpm_replaces] = Array(packager.buildizer_conf['fpm_replaces'])
+          params[:fpm_provides] = Array(packager.buildizer_conf['fpm_provides'])
+          params[:fpm_depends] = Array(packager.buildizer_conf['fpm_depends'])
+          params[:fpm_description] = packager.buildizer_conf['fpm_description']
+          params[:fpm_url] = packager.buildizer_conf['fpm_url']
         end
       end
 
       def do_merge_params(into, params)
         super.tap do |res|
           res[:fpm_script] = into[:fpm_script] + Array(params['fpm_script'])
-          res[:fpm_config_files] = into[:fpm_config_files].merge params['fpm_config_files'].to_h
-          res[:fpm_files] = into[:fpm_files].merge params['fpm_files'].to_h
-        end
-      end
-
-      def merge_os_params(os_name, **kwargs, &blk)
-        super(os_name, **kwargs) do |into, params|
-          yield into, params if block_given?
-          cannot_redefine_package_params!(params, redefine_for: "os '#{os_name}'")
-        end
-      end
-
-      def merge_os_version_params(os_name, os_version, **kwargs, &blk)
-        super(os_name, os_version, **kwargs) do |into, params|
-          yield into, params if block_given?
-          cannot_redefine_package_params!(params,
-                                          redefine_for: "os version '#{os_name}-#{os_version}'")
-        end
-      end
-
-      def merge_base_target_params(target, target_package_name, target_package_version, **kwargs, &blk)
-        super(target, target_package_name, target_package_version, **kwargs) do |into, params|
-          yield into, params if block_given?
-          cannot_redefine_package_params!(params, redefine_for: "target '#{target}'")
+          res[:fpm_config_files] = into[:fpm_config_files].merge(params['fpm_config_files'].to_h)
+          res[:fpm_files] = into[:fpm_files].merge(params['fpm_files'].to_h)
+          res[:fpm_conflicts] = (into[:fpm_conflicts] + Array(params['fpm_conflicts'])).uniq
+          res[:fpm_replaces] = (into[:fpm_replaces] + Array(params['fpm_replaces'])).uniq
+          res[:fpm_provides] = (into[:fpm_provides] + Array(params['fpm_provides'])).uniq
+          res[:fpm_depends] = (into[:fpm_depends] + Array(params['fpm_depends'])).uniq
+          res[:fpm_description] = params['fpm_description'] || into[:fpm_description]
+          res[:fpm_url] = params['fpm_url'] || into[:fpm_url]
         end
       end
 
@@ -79,6 +48,7 @@ module Buildizer
                        message: ["either of fpm_files or fpm_config_files ",
                                  "required in #{build_type} build_type"].join
         end
+        _required_params! :package_version, params
       end
 
       def build_instructions(target)
@@ -96,9 +66,8 @@ module Buildizer
               message: "unknown fpm_script event #{_when.inspect}"
             ) unless FPM_SCRIPT_EVENTS.include? _when
             res[_when] ||= {fpm_option: "--#{_when.split('_').join('-')}",
-                            file: docker.image_runtime_build_path(target.image)
-                                        .join("fpm_#{_when}.sh"),
-                            container_file: Pathname.new('/package/build').join("fpm_#{_when}.sh"),
+                            file: target.image_extra_path.join("fpm_#{_when}.sh"),
+                            container_file: docker.container_extra_path.join("fpm_#{_when}.sh"),
                             cmd: []}
             res[_when][:cmd] += cmd
           end
@@ -121,8 +90,16 @@ module Buildizer
          "--iteration=#{release}",
          *fpm_script.values.map {|desc| "#{desc[:fpm_option]}=#{desc[:container_file]}"},
          *Array(target.image.fpm_extra_params),
-         *target.fpm_config_files.values.map {|p| "--config-files=#{p}"},
-         *target.fpm_files.merge(target.fpm_config_files).map {|p1, p2| "#{p1}=#{p2}"}].join(' ')
+         (target.maintainer ? "--maintainer=\"#{target.maintainer}\"" : nil),
+         (target.fpm_description ? "--description=\"#{target.fpm_description}\"" : nil),
+         (target.fpm_url ? "--url=\"#{target.fpm_url}\"" : nil),
+         *target.fpm_conflicts.map{|pkg| "--conflicts=#{pkg}"},
+         *target.fpm_replaces.map{|pkg| "--replaces=#{pkg}"},
+         *target.fpm_provides.map{|pkg| "--provides=#{pkg}"},
+         *target.fpm_depends.map{|pkg| "--depends=#{pkg}"},
+         *target.fpm_config_files_expand.keys.map {|p| "--config-files=#{p}"},
+         *target.fpm_files_expand.merge(target.fpm_config_files_expand).map {|p1, p2| "#{p2}=#{p1}"},
+        ].compact.join(' ')
       end
     end # Fpm
   end # Builder