conjur-debify 1.11.4 → 2.1.0

data/lib/conjur/debify.rb

@@ -4,6 +4,7 @@ require 'fileutils'
 require 'gli'
 require 'json'
 require 'base64'
+require 'tmpdir'
 
 require 'conjur/debify/utils'
 
@@ -12,6 +13,8 @@ require 'active_support/core_ext'
 
 include GLI::App
 
+DEFAULT_FILE_TYPE = "deb"
+
 config_file '.debifyrc'
 
 desc 'Set an environment variable (e.g. TERM=xterm) when starting a container'
@@ -49,7 +52,7 @@ module DebugMixin
     if a.length == 2 && a[0].is_a?(Symbol)
       debug a.last
     else
-       a.each do |line|
+      a.each do |line|
         begin
           line = JSON.parse(line)
           line.keys.each do |k|
@@ -79,7 +82,7 @@ def detect_version
     base_version = File.read("VERSION").strip
     commits_since = `git log #{base_commit}..HEAD --pretty='%h'`.split("\n").size
     hash = `git rev-parse --short HEAD`.strip
-    [ [ base_version, commits_since ].join('.'), hash ].join("-")
+    [[base_version, commits_since].join('.'), hash].join("-")
   else
     `git describe --long --tags --abbrev=7 --match 'v*.*.*' | sed -e 's/^v//'`.strip.tap do |version|
       raise "No Git version (tag) for project" if version.empty?
@@ -88,7 +91,13 @@ def detect_version
 end
 
 def git_files
-  (`git ls-files -z`.split("\x0") + ['Gemfile.lock']).uniq
+  files = (`git ls-files -z`.split("\x0") + ['Gemfile.lock']).uniq
+  # Since submodule directories are listed, but are not files, we remove them.
+  # Currently, `conjur-project-config` is the only submodule in Conjur, and it
+  # can safely be removed because it's a developer-only tool.  If we add another
+  # submodule in the future needed for production, we'll need to update this
+  # code.  But YAGNI for now.
+  files.select { |f| File.file?(f) }
 end
 
 def login_to_registry(appliance_image_id)
@@ -124,15 +133,15 @@ DESC
 arg_name "project-name -- <fpm-arguments>"
 command "clean" do |c|
   c.desc "Set the current working directory"
-  c.flag [ :d, "dir" ]
+  c.flag [:d, "dir"]
 
   c.desc "Ignore (don't delete) a file or directory"
-  c.flag [ :i, :ignore ]
+  c.flag [:i, :ignore]
 
   c.desc "Force file deletion even if if this doesn't look like a Jenkins environment"
-  c.switch [ :force ]
+  c.switch [:force]
 
-  c.action do |global_options,cmd_options,args|
+  c.action do |global_options, cmd_options, args|
     def looks_like_jenkins?
       require 'etc'
       Etc.getlogin == 'jenkins' && ENV['BUILD_NUMBER']
@@ -143,10 +152,10 @@ command "clean" do |c|
     if !perform_deletion
       $stderr.puts "No --force, and this doesn't look like Jenkins. I won't actually delete anything"
     end
-    @ignore_list = Array(cmd_options[:ignore]) + [ '.', '..', '.git' ]
+    @ignore_list = Array(cmd_options[:ignore]) + ['.', '..', '.git']
 
     def ignore_file? f
-      @ignore_list.find{|ignore| f.index(ignore) == 0}
+      @ignore_list.find { |ignore| f.index(ignore) == 0 }
     end
 
     dir = cmd_options[:dir] || '.'
@@ -159,16 +168,16 @@ command "clean" do |c|
       end
       find_files.compact!
       delete_files = (find_files - git_files)
-      delete_files.delete_if{|file|
+      delete_files.delete_if { |file|
         File.directory?(file) || ignore_file?(file)
       }
       if perform_deletion
         image = Docker::Image.create 'fromImage' => "alpine:3.3"
         options = {
-          'Cmd'   => [ "sh", "-c", "while true; do sleep 1; done" ],
+          'Cmd' => ["sh", "-c", "while true; do sleep 1; done"],
           'Image' => image.id,
           'Binds' => [
-            [ dir, "/src" ].join(':'),
+            [dir, "/src"].join(':'),
           ]
         }
         options['Privileged'] = true if Docker.version['Version'] >= '1.10.0'
@@ -179,7 +188,7 @@ command "clean" do |c|
             puts file
 
             file = "/src/#{file}"
-            cmd = [ "rm", "-f", file ]
+            cmd = ["rm", "-f", file]
 
             stdout, stderr, status = container.exec cmd, &DebugMixin::DOCKER
             $stderr.puts "Failed to delete #{file}" unless status == 0
@@ -196,6 +205,17 @@ command "clean" do |c|
   end
 end
 
+def copy_packages_from_container(container, package_name, dev_package_name)
+  Conjur::Debify::Utils.copy_from_container container, "/src/#{package_name}"
+  puts "#{package_name}"
+  begin
+    Conjur::Debify::Utils.copy_from_container container, "/dev-pkg/#{dev_package_name}"
+    puts "#{dev_package_name}"
+  rescue Docker::Error::NotFoundError
+    warn "#{dev_package_name} not found. The package might not have any development dependencies."
+  end
+end
+
 desc "Build a debian package for a project"
 long_desc <<DESC
 The package is built using fpm (https://github.com/jordansissel/fpm).
@@ -220,15 +240,21 @@ DESC
 arg_name "project-name -- <fpm-arguments>"
 command "package" do |c|
   c.desc "Set the current working directory"
-  c.flag [ :d, "dir" ]
+  c.flag [:d, "dir"]
+
+  c.desc "Set the output file type of the fpm command (e.g rpm)"
+  c.flag [:o, :output]
 
   c.desc "Specify the deb version; by default, it's read from the VERSION file"
-  c.flag [ :v, :version ]
+  c.flag [:v, :version]
 
   c.desc "Specify a custom Dockerfile.fpm"
-  c.flag [ :dockerfile]
+  c.flag [:dockerfile]
+
+  c.desc "Specify files to add to the FPM image that are not included from the git repo"
+  c.flag [:'additional-files']
 
-  c.action do |global_options,cmd_options,args|
+  c.action do |global_options, cmd_options, args|
     raise "project-name is required" unless project_name = args.shift
 
     fpm_args = []
@@ -241,30 +267,62 @@ command "package" do |c|
     dir = cmd_options[:dir] || '.'
     pwd = File.dirname(__FILE__)
 
-    fpm_image = Docker::Image.build_from_dir File.expand_path('fpm', File.dirname(__FILE__)), tag: "debify-fpm", &DebugMixin::DOCKER
+    additional_files = []
+    if cmd_options[:'additional-files']
+      additional_files = cmd_options[:'additional-files'].split(',').map(&:strip)
+    end
+
+    begin
+      tries ||= 2
+      fpm_image = Docker::Image.build_from_dir File.expand_path('fpm', File.dirname(__FILE__)), tag: "debify-fpm", &DebugMixin::DOCKER
+    rescue
+      image_id = File.readlines(File.expand_path('fpm/Dockerfile', File.dirname(__FILE__)))
+                     .find { | line | line =~ /^FROM/ }
+                     .split(' ')
+                     .last
+      login_to_registry image_id
+      retry unless (tries -= 1).zero?
+    end
     DebugMixin.debug_write "Built base fpm image '#{fpm_image.id}'\n"
     dir = File.expand_path(dir)
+
     Dir.chdir dir do
       version = cmd_options[:version] || detect_version
-      dockerfile_path = cmd_options[:dockerfile] || File.expand_path("debify/Dockerfile.fpm", pwd)
-      dockerfile = File.read(dockerfile_path)
 
-      output = StringIO.new
-      Gem::Package::TarWriter.new(output) do |tar|
-        git_files.each do |fname|
-          stat = File.stat(fname)
-          tar.add_file(fname, stat.mode) { |tar_file| tar_file.write(File.read(fname)) }
-        end
-        tar.add_file('Dockerfile', 0640) { |tar_file| tar_file.write dockerfile.gsub("@@image@@", fpm_image.id) }
+      # move git files and Dockerfile to temp dir to make deb from
+      # we do this to avoid adding "non-git" files
+      # that aren't mentioned in the dockerignore to the deb
+      temp_dir = Dir.mktmpdir
+      DebugMixin.debug_write "Copying git files to tmp dir '#{temp_dir}'\n"
+      (git_files + additional_files).each do |fname|
+        original_file = File.join(dir, fname)
+        destination_path = File.join(temp_dir, fname)
+        FileUtils.mkdir_p(File.dirname(destination_path))
+        FileUtils.cp(original_file, destination_path)
       end
-      output.rewind
 
-      image = Docker::Image.build_from_tar output, &DebugMixin::DOCKER
+      # rename specified dockerfile to 'Dockerfile' during copy, incase name is different
+      dockerfile_path = cmd_options[:dockerfile] || File.expand_path("debify/Dockerfile.fpm", pwd)
+      temp_dockerfile = File.join(temp_dir, "Dockerfile")
+
+      # change image variable in specified Dockerfile
+      dockerfile = File.read(dockerfile_path)
+      replace_image = dockerfile.gsub("@@image@@", fpm_image.id)
+      File.open(temp_dockerfile, "w") { |file| file.puts replace_image }
+
+      # build image from project being debified dir
+      image = Docker::Image.build_from_dir temp_dir, &DebugMixin::DOCKER
 
       DebugMixin.debug_write "Built fpm image '#{image.id}' for project #{project_name}\n"
 
+      container_cmd_options = [project_name, version]
+
+      # Set the output file type if present
+      file_type = cmd_options[:output] || DEFAULT_FILE_TYPE
+      container_cmd_options << "--file-type=#{file_type}"
+
       options = {
-        'Cmd'   => [ project_name, version ] + fpm_args,
+        'Cmd' => container_cmd_options + fpm_args,
         'Image' => image.id
       }
       options['Privileged'] = true if Docker.version['Version'] >= '1.10.0'
@@ -276,15 +334,22 @@ command "package" do |c|
         status = container.wait
         raise "Failed to package #{project_name}" unless status['StatusCode'] == 0
 
-        pkg = "conjur-#{project_name}_#{version}_amd64.deb"
-        dev_pkg = "conjur-#{project_name}-dev_#{version}_amd64.deb"
-        Conjur::Debify::Utils.copy_from_container container, "/src/#{pkg}"
-        puts "#{pkg}"
-        begin
-          Conjur::Debify::Utils.copy_from_container container, "/dev-pkg/#{dev_pkg}"
-          puts "#{dev_pkg}"
-        rescue Docker::Error::NotFoundError
-          warn "#{dev_pkg} not found. The package might not have any development dependencies."
+        if file_type == "deb"
+          # Copy deb packages
+          copy_packages_from_container(
+            container,
+            "conjur-#{project_name}_#{version}_amd64.deb",
+            "conjur-#{project_name}-dev_#{version}_amd64.deb"
+          )
+        elsif file_type == "rpm"
+          # Copy rpm packages
+          # The rpm builder replaces dashes with underscores in the version
+          rpm_version = version.tr('-', '_')
+          copy_packages_from_container(
+            container,
+            "conjur-#{project_name}-#{rpm_version}-1.x86_64.rpm",
+            "conjur-#{project_name}-dev-#{rpm_version}-1.x86_64.rpm"
+          )
         end
       ensure
         container.delete(force: true)
@@ -308,10 +373,10 @@ end
 
 def network_options(cmd)
   cmd.desc "Specify link for test container"
-  cmd.flag [ :l, :link ], :multiple => true
-  
+  cmd.flag [:l, :link], :multiple => true
+
   cmd.desc 'Attach to the specified network'
-  cmd.flag [ :n, :net ]
+  cmd.flag [:n, :net]
 end
 
 def short_id(id)
@@ -327,7 +392,7 @@ end
 # instead. (Docker doesn't add full container ids as network aliases,
 # only short ids).
 def shorten_source_id(link)
-  src,dest = link.split(':')
+  src, dest = link.split(':')
   src && dest ? "#{short_id(src)}:#{dest}" : link
 end
 
@@ -377,32 +442,32 @@ DESC
 arg_name "project-name test-script"
 command "test" do |c|
   c.desc "Set the current working directory"
-  c.flag [ :d, :dir ]
+  c.flag [:d, :dir]
 
   c.desc "Keep the Conjur appliance container after the command finishes"
   c.default_value false
-  c.switch [ :k, :keep ]
+  c.switch [:k, :keep]
 
   c.desc "Image name"
   c.default_value "registry.tld/conjur-appliance-cuke-master"
-  c.flag [ :i, :image ]
+  c.flag [:i, :image]
 
   c.desc "Image tag, e.g. 4.5-stable, 4.6-stable"
-  c.flag [ :t, "image-tag"]
+  c.flag [:t, "image-tag"]
 
   c.desc "'docker pull' the Conjur container image"
   c.default_value true
-  c.switch [ :pull ]
+  c.switch [:pull]
 
   c.desc "Specify the deb version; by default, it's read from the VERSION file"
-  c.flag [ :v, :version ]
+  c.flag [:v, :version]
 
   c.desc "Specify volume for test container"
-  c.flag [ :'volumes-from' ], :multiple => true
+  c.flag [:'volumes-from'], :multiple => true
 
   network_options(c)
-  
-  c.action do |global_options,cmd_options,args|
+
+  c.action do |global_options, cmd_options, args|
     raise "project-name is required" unless project_name = args.shift
     raise "test-script is required" unless test_script = args.shift
     raise "Received extra command-line arguments" if args.shift
@@ -415,7 +480,7 @@ command "test" do |c|
 
     Dir.chdir dir do
       image_tag = cmd_options["image-tag"] or raise "image-tag is required"
-      appliance_image_id = [ cmd_options[:image], image_tag ].join(":")
+      appliance_image_id = [cmd_options[:image], image_tag].join(":")
       version = cmd_options[:version] || detect_version
       package_name = "conjur-#{project_name}_#{version}_amd64.deb"
       dev_package_name = "conjur-#{project_name}-dev_#{version}_amd64.deb"
@@ -423,7 +488,7 @@ command "test" do |c|
       raise "#{test_script} does not exist or is not a file" unless File.file?(test_script)
 
       begin
-        tries ||=2
+        tries ||= 2
         Docker::Image.create 'fromImage' => appliance_image_id, &DebugMixin::DOCKER if cmd_options[:pull]
       rescue
         login_to_registry appliance_image_id
@@ -462,7 +527,7 @@ RUN touch /etc/service/conjur/down
       packages << dev_package_name if File.exist? dev_package_name
 
       begin
-        tries ||=2
+        tries ||= 2
         appliance_image = build_test_image(appliance_image_id, project_name, packages)
       rescue
         login_to_registry appliance_image_id
@@ -483,29 +548,29 @@ RUN touch /etc/service/conjur/down
         ] + global_options[:env],
         'HostConfig' => {
           'Binds' => [
-            [ dir, "/src/#{project_name}" ].join(':')
+            [dir, "/src/#{project_name}"].join(':')
           ]
         }
       }
       host_config = options['HostConfig']
-      
+
       host_config['Privileged'] = true if Docker.version['Version'] >= '1.10.0'
       host_config['VolumesFrom'] = cmd_options[:'volumes-from'] if cmd_options[:'volumes-from'] && !cmd_options[:'volumes-from'].empty?
 
       add_network_config(options, cmd_options)
-      
+
       if global_options[:'local-bundle']
         host_config['Binds']
-          .push([ vendor_dir, "/src/#{project_name}/vendor" ].join(':'))
-          .push([ dot_bundle_dir, "/src/#{project_name}/.bundle" ].join(':'))
+          .push([vendor_dir, "/src/#{project_name}/vendor"].join(':'))
+          .push([dot_bundle_dir, "/src/#{project_name}/.bundle"].join(':'))
       end
 
-      container = Docker::Container.create(options.tap {|o| DebugMixin.debug_write "creating container with options #{o.inspect}"})
+      container = Docker::Container.create(options.tap { |o| DebugMixin.debug_write "creating container with options #{o.inspect}" })
 
       begin
         DebugMixin.debug_write "Testing #{project_name} in container #{container.id}\n"
 
-        spawn("docker logs -f #{container.id}", [ :out, :err ] => $stderr).tap do |pid|
+        spawn("docker logs -f #{container.id}", [:out, :err] => $stderr).tap do |pid|
           Process.detach pid
         end
         container.start!
@@ -556,29 +621,29 @@ Once in the container, use "/opt/conjur/evoke/bin/dev-install" to install the de
 DESC
 command "sandbox" do |c|
   c.desc "Set the current working directory"
-  c.flag [ :d, :dir ]
+  c.flag [:d, :dir]
 
   c.desc "Image name"
   c.default_value "registry.tld/conjur-appliance-cuke-master"
-  c.flag [ :i, :image ]
+  c.flag [:i, :image]
 
   c.desc "Image tag, e.g. 4.5-stable, 4.6-stable"
-  c.flag [ :t, "image-tag"]
+  c.flag [:t, "image-tag"]
 
   c.desc "Bind another source directory into the container. Use <src>:<dest>, where both are full paths."
-  c.flag [ :"bind" ], :multiple => true
+  c.flag [:"bind"], :multiple => true
 
   c.desc "'docker pull' the Conjur container image"
   c.default_value false
-  c.switch [ :pull ]
+  c.switch [:pull]
 
   network_options(c)
 
   c.desc "Specify volume for container"
-  c.flag [ :'volumes-from' ], :multiple => true
+  c.flag [:'volumes-from'], :multiple => true
 
   c.desc "Expose a port from the container to host. Use <host>:<container>."
-  c.flag [ :p, :port ], :multiple => true
+  c.flag [:p, :port], :multiple => true
 
   c.desc 'Run dev-install in /src/<project-name>'
   c.default_value false
@@ -589,9 +654,9 @@ command "sandbox" do |c|
   c.switch [:kill]
 
   c.desc 'A command to run in the sandbox'
-  c.flag [ :c, :command ]
-  
-  c.action do |global_options,cmd_options,args|
+  c.flag [:c, :command]
+
+  c.action do |global_options, cmd_options, args|
     raise "Received extra command-line arguments" if args.shift
 
     dir = cmd_options[:dir] || '.'
@@ -601,11 +666,11 @@ command "sandbox" do |c|
 
     Dir.chdir dir do
       image_tag = cmd_options["image-tag"] or raise "image-tag is required"
-      appliance_image_id = [ cmd_options[:image], image_tag ].join(":")
+      appliance_image_id = [cmd_options[:image], image_tag].join(":")
 
       appliance_image = if cmd_options[:pull]
         begin
-          tries ||=2
+          tries ||= 2
           Docker::Image.create 'fromImage' => appliance_image_id, &DebugMixin::DOCKER if cmd_options[:pull]
         rescue
           login_to_registry appliance_image_id
@@ -635,26 +700,26 @@ command "sandbox" do |c|
 
       options['HostConfig'] = host_config = {}
       host_config['Binds'] = [
-        [ File.expand_path(".ssh/id_rsa", ENV['HOME']), "/root/.ssh/id_rsa", 'ro' ].join(':'),
-        [ dir, "/src/#{project_name}" ].join(':'),
+        [File.expand_path(".ssh/id_rsa", ENV['HOME']), "/root/.ssh/id_rsa", 'ro'].join(':'),
+        [dir, "/src/#{project_name}"].join(':'),
       ] + Array(cmd_options[:bind])
 
       if global_options[:'local-bundle']
         host_config['Binds']
-          .push([ vendor_dir, "/src/#{project_name}/vendor" ].join(':'))
-          .push([ dot_bundle_dir, "/src/#{project_name}/.bundle" ].join(':'))
+          .push([vendor_dir, "/src/#{project_name}/vendor"].join(':'))
+          .push([dot_bundle_dir, "/src/#{project_name}/.bundle"].join(':'))
       end
 
       host_config['Privileged'] = true if Docker.version['Version'] >= '1.10.0'
       host_config['VolumesFrom'] = cmd_options[:'volumes-from'] unless cmd_options[:'volumes-from'].empty?
-      
+
       add_network_config(options, cmd_options)
 
       unless cmd_options[:port].empty?
         port_bindings = Hash.new({})
         cmd_options[:port].each do |mapping|
           hport, cport = mapping.split(':')
-          port_bindings["#{cport}/tcp"] = [{ 'HostPort' => hport }]
+          port_bindings["#{cport}/tcp"] = [{'HostPort' => hport}]
         end
         host_config['PortBindings'] = port_bindings
       end
@@ -664,7 +729,7 @@ command "sandbox" do |c|
         previous.delete(:force => true) if previous
       end
 
-      container = Docker::Container.create(options.tap {|o| DebugMixin.debug_write "creating container with options #{o.inspect}"})
+      container = Docker::Container.create(options.tap { |o| DebugMixin.debug_write "creating container with options #{o.inspect}" })
       $stdout.puts container.id
       container.start!
 
@@ -700,23 +765,27 @@ DESC
 arg_name "distribution project-name"
 command "publish" do |c|
   c.desc "Set the current working directory"
-  c.flag [ :d, :dir ]
+  c.flag [:d, :dir]
 
   c.desc "Specify the deb package version; by default, it's computed automatically"
-  c.flag [ :v, :version ]
+  c.flag [:v, :version]
 
   c.desc "Component to publish to, either 'stable' or the name of the git branch"
-  c.flag [ :c, :component ]
+  c.flag [:c, :component]
 
   c.desc "Artifactory URL to publish to"
   c.default_value "https://conjurinc.jfrog.io/conjurinc"
-  c.flag [ :u, :url]
+  c.flag [:u, :url]
 
   c.desc "Artifactory Debian repo to publish package to"
   c.default_value "debian-private"
-  c.flag [ :r, :repo]
+  c.flag [:r, :repo]
+
+  c.desc "Artifactory RPM repo to publish package to"
+  c.default_value "redhat-private"
+  c.flag ['rpm-repo']
 
-  c.action do |global_options,cmd_options,args|
+  c.action do |global_options, cmd_options, args|
     require 'conjur/debify/action/publish'
     raise "distribution is required" unless distribution = args.shift
     raise "project-name is required" unless project_name = args.shift
@@ -729,8 +798,8 @@ end
 desc "Auto-detect and print the repository version"
 command "detect-version" do |c|
   c.desc "Set the current working directory"
-  c.flag [ :d, :dir ]
-  c.action do |global_options,cmd_options,args|
+  c.flag [:d, :dir]
+  c.action do |global_options, cmd_options, args|
     raise "Received extra command-line arguments" if args.shift
 
     dir = cmd_options[:dir] || '.'
@@ -747,7 +816,7 @@ end
 desc 'Show the given configuration'
 arg_name 'configuration'
 command 'config' do |c|
-  c.action do |_,_,args|
+  c.action do |_, _, args|
     raise 'no configuration provided' unless config = args.shift
     raise "Received extra command-line arguments" if args.shift
 
@@ -758,7 +827,7 @@ command 'config' do |c|
 end
 
 
-pre do |global,command,options,args|
+pre do |global, command, options, args|
   # Pre logic here
   # Return true to proceed; false to abort and not call the
   # chosen command
@@ -767,7 +836,7 @@ pre do |global,command,options,args|
   true
 end
 
-post do |global,command,options,args|
+post do |global, command, options, args|
   # Post logic here
   # Use skips_post before a command to skip this
   # block on that command only

data/lib/conjur/fpm/Dockerfile

@@ -1,23 +1,19 @@
 # Build from the same version of ubuntu as phusion/baseimage
-FROM ubuntu:18.04
+FROM cyberark/phusion-ruby-fips:latest
 
 RUN apt-get update -y && \
+    apt-get dist-upgrade -y && \
     apt-get install -y build-essential \
                        git \
                        libffi-dev \
-                       libpq5 \
-                       libpq-dev \
-                       ruby2.5 \
-                       ruby2.5-dev
+                       rpm
 
-RUN gem install --no-document bundler:1.17.3 \
-                              fpm
+RUN gem install --no-document fpm
 
 ENV GEM_HOME /usr/local/bundle
 ENV BUNDLE_PATH="$GEM_HOME" \
     BUNDLE_BIN="$GEM_HOME/bin" \
-    BUNDLE_SILENCE_ROOT_WARNING=1 \
-    BUNDLE_APP_CONFIG="$GEM_HOME"
+    BUNDLE_SILENCE_ROOT_WARNING=1
 ENV PATH $BUNDLE_BIN:$PATH
 RUN mkdir -p "$GEM_HOME" "$BUNDLE_BIN" && \
     chmod 777 "$GEM_HOME" "$BUNDLE_BIN"