pipedawg 0.3.1 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 041e80c36e4a1f30695c0d60233c8fcc3261ac734dee7d0770dde11ad1e0ac71
-  data.tar.gz: ee48577c9c00e3ef4f76cabe22895d435b26bb10cd6dd4ff5a950dbaebaaf5bc
+  metadata.gz: b2e0ba1c718b0efe498f222720490aeb2566a6f5c4e6fb1f0b56e5e9874669a5
+  data.tar.gz: 5545deb6f783e71bb5b35bb6633aed7939a9ac0e74aefb50200020976c1b6489
 SHA512:
-  metadata.gz: 83e2ef040fbdbec747968ca61d58079e720dd32cea77fd0dc0b39f451dab618e247cf85617f6f6f6dbaf881263e90c4b2f7b90b3df96d5c60e6a338db5b007d1
-  data.tar.gz: 34cccd812da30055338afc7368a5f62f46c0df7ba8e7dc7c21e9590fdcd4a5d398df30be22f0e5aac6c7a9a3944f4683b1b96be79da9b75276aca89ee13e8d89
+  metadata.gz: 17fbfd9bb667f808c44e636d7133fa379d115b650102162d35b5e0fd7b6594466493663c7193ef8092d3e55c2480737bbf4e4bf5923260096ae7706200320d04
+  data.tar.gz: 338192888145922229beba1e38fa3794e2f1d7d4e197987de1b5ae7425df29f2239f47a011846b10df20723d43c93a26d26813755a33d2b193780b0812429f74

data/README.md

@@ -42,13 +42,13 @@ gem_job = Pipedawg::Job.new(
   script: ['bundle install', 'gem build *.gemspec']
 )
 
-kaniko_job = Pipedawg::KanikoJob.new(
+kaniko_build_job = Pipedawg::KanikoBuildJob.new(
   'build:kaniko',
   {needs: ['build:gem'], retry: 2},
   {context:'${CI_PROJECT_DIR}/docker',external_files: {'*.gem':'gems'}}
 )
 
-pipeline = Pipedawg::Pipeline.new 'build:image', jobs: [gem_job, kaniko_job]
+pipeline = Pipedawg::Pipeline.new 'build:image', jobs: [gem_job, kaniko_build_job]
 puts pipeline.to_yaml
 ```
 

data/lib/pipedawg/helm_copy_job.rb

@@ -7,7 +7,7 @@ module Pipedawg
 
     def initialize(name = 'build', opts = {}, helm_opts = {})
       @helm_opts = {
-        chart: name,
+        chart: name, debug: true,
         destinations: [{ user: nil, password: nil, url: nil }],
         helm: 'helm',
         image: { entrypoint: [''], name: 'alpine/helm' },
@@ -17,13 +17,21 @@ module Pipedawg
       update
     end
 
-    def update
+    def update # rubocop:disable Metrics/AbcSize
       opts[:image] = helm_opts[:image] if helm_opts[:image]
-      opts[:script] = [] + pull + (helm_opts[:destinations].map { |d| push(d) }).flatten(1)
+      opts[:script] = debug + pull + (helm_opts[:destinations].map { |d| push(d) }).flatten(1)
     end
 
     private
 
+    def debug
+      if helm_opts[:debug]
+        Pipedawg::Util.echo_proxy_vars
+      else
+        []
+      end
+    end
+
     def pull
       case helm_opts[:url]
       when nil

data/lib/pipedawg/{kaniko_job.rb → kaniko_build_job.rb}

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 module Pipedawg
-  # kaniko_job class
-  class KanikoJob < Job
+  # kaniko_build_job class
+  class KanikoBuildJob < Job
     attr_accessor :kaniko_opts
 
     def initialize(name = 'build', opts = {}, kaniko_opts = {}) # rubocop:disable Metrics/MethodLength
@@ -11,13 +11,12 @@ module Pipedawg
         config: {
           '$CI_REGISTRY': { username: '$CI_REGISTRY_USER', password: '$CI_REGISTRY_PASSWORD' }
         },
-        config_file: '/kaniko/.docker/config.json', context: '${CI_PROJECT_DIR}',
-        destinations: [], dockerfile: 'Dockerfile', executor: '/kaniko/executor',
-        external_files: {}, flags: [], ignore_paths: [], insecure_registries: [],
-        image: { entrypoint: [''], name: 'gcr.io/kaniko-project/executor:debug' },
-        options: {}, registry_certificates: {}, registry_mirrors: [],
-        skip_tls_verify_registry: [], trusted_ca_cert_source_files: [],
-        trusted_ca_cert_target_file: '/kaniko/ssl/certs/ca-certificates.crt'
+        config_file: '/kaniko/.docker/config.json', context: '${CI_PROJECT_DIR}', debug: true,
+        destinations: [], dockerfile: 'Dockerfile', executor: '/kaniko/executor', external_files: {},
+        flags: [], ignore_paths: [], insecure_registries: [],
+        image: { entrypoint: [''], name: 'gcr.io/kaniko-project/executor:debug' }, options: {},
+        registry_certificates: {}, registry_mirrors: [], skip_tls_verify_registry: [],
+        trusted_ca_cert_source_files: [], trusted_ca_cert_target_file: '/kaniko/ssl/certs/ca-certificates.crt'
       }.merge(kaniko_opts)
       super name, opts
       update
@@ -26,11 +25,19 @@ module Pipedawg
     def update # rubocop:disable Metrics/AbcSize
       require 'json'
       opts[:image] = kaniko_opts[:image] if kaniko_opts[:image]
-      opts[:script] = config + cert_copies + file_copies + Array(kaniko_cmd)
+      opts[:script] = debug + config + cert_copies + file_copies + Array(kaniko_cmd)
     end
 
     private
 
+    def debug
+      if kaniko_opts[:debug]
+        Pipedawg::Util.echo_proxy_vars
+      else
+        []
+      end
+    end
+
     def config
       ["echo #{kaniko_opts[:config].to_json.inspect} > \"#{kaniko_opts[:config_file]}\""]
     end

data/lib/pipedawg/qualys_scan_job.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module Pipedawg
+  # qualys_scan_job class
+  class QualysScanJob < Job # rubocop:disable Metrics/ClassLength
+    attr_accessor :qualys_opts
+
+    def initialize(name = 'build', opts = {}, qualys_opts = {})
+      @qualys_opts = {
+        acceptable_risk: '${QUALYS_ACCEPTABLE_IMAGE_RISK}',
+        artifacts: { expire_in: '1 month', paths: ['software.json', 'vulnerabilities.json'], when: 'always' },
+        config: { '$CI_REGISTRY': { username: '$CI_REGISTRY_USER', password: '$CI_REGISTRY_PASSWORD' } },
+        debug: true, gateway: '${QUALYS_GATEWAY}', image: nil, password: '${QUALYS_PASSWORD}', rules: nil,
+        scan_image: '${QUALYS_IMAGE}', scan_target_prefix: 'qualys_scan_target', tags: nil, user: '${QUALYS_USERNAME}',
+        variables: { GIT_STRATEGY: 'clone' }
+      }.merge(qualys_opts)
+      super name, opts
+      update
+    end
+
+    def update # rubocop:disable Metrics/AbcSize
+      require 'json'
+      opts[:artifacts] = qualys_opts[:artifacts] if qualys_opts[:artifacts]
+      opts[:image] = qualys_opts[:image]
+      opts[:rules] = qualys_opts[:rules] if qualys_opts[:rules]
+      opts[:tags] = qualys_opts[:tags] if qualys_opts[:tags]
+      opts[:variables] = qualys_opts[:variables] if qualys_opts[:variables]
+      opts[:script] =
+        debug + config + image + clean_config + token + scan_start + scan_complete + artifacts + severities + outputs
+    end
+
+    private
+
+    def debug # rubocop:disable Metrics/MethodLength
+      if qualys_opts[:debug]
+        Pipedawg::Util.echo_proxy_vars + [
+          'echo Qualys settings:', "echo   Qualys gateway: \"#{qualys_opts[:gateway]}\"",
+          "echo   Qualys username: \"#{qualys_opts[:user]}\"",
+          "if [ \"#{qualys_opts[:password]}\" != '' ]; then " \
+          'echo   Qualys password is not empty; else ' \
+          'echo   Qualys password is not set; exit 1; fi'
+        ]
+      else
+        []
+      end
+    end
+
+    def config
+      ['export CONFIG=$(mktemp -d)', "echo #{qualys_opts[:config].to_json.inspect} > \"${CONFIG}/config.json\""]
+    end
+
+    def image
+      [
+        "image_target=\"#{qualys_opts[:scan_target_prefix]}:$(echo #{qualys_opts[:scan_image]} | sed 's/^[^/]*\\///'| sed 's/[:/]/-/g')\"", # rubocop:disable Layout/LineLength
+        "docker --config=\"${CONFIG}\" pull \"#{qualys_opts[:scan_image]}\"",
+        "docker image tag \"#{qualys_opts[:scan_image]}\" \"${image_target}\"",
+        "image_id=$(docker inspect --format=\"{{index .Id}}\" \"#{qualys_opts[:scan_image]}\" | cut -c8-19)",
+        'echo "Image ID: ${image_id}"'
+      ]
+    end
+
+    def clean_config
+      [
+        'rm -f "${CONFIG}/config.json"',
+        'rmdir "${CONFIG}"'
+      ]
+    end
+
+    def token
+      ["token=$(curl -s --location --request POST \"https://#{qualys_opts[:gateway]}/auth\" --header \"Content-Type: application/x-www-form-urlencoded\" --data-urlencode \"username=#{qualys_opts[:user]}\" --data-urlencode \"password=#{qualys_opts[:password]}\" --data-urlencode \"token=true\")"] # rubocop:disable Layout/LineLength
+    end
+
+    def scan_start
+      [
+        'while true; do ' \
+        "result=$(curl -s -o /dev/null -w ''%{http_code}'' --location --request GET \"https://#{qualys_opts[:gateway]}/csapi/v1.2/images/$image_id\" --header \"Authorization: Bearer $token\"); " + # rubocop:disable Layout/LineLength, Style/FormatStringToken
+          'echo "Waiting for scan to start..."; ' \
+          'echo "  Result: ${result}"; ' \
+          'if [ "${result}" = "200" ]; then break; fi; ' \
+          'sleep 10; done'
+      ]
+    end
+
+    def scan_complete
+      [
+        'while true; do ' \
+        "result=$(curl -s --location --request GET \"https://#{qualys_opts[:gateway]}/csapi/v1.2/images/$image_id\" --header \"Authorization: Bearer $token\" | jq -r '.scanStatus'); " + # rubocop:disable Layout/LineLength
+          'echo "Waiting for scan to complete..."; ' \
+          'echo "  Result: ${result}"; ' \
+          'if [ "${result}" = "SUCCESS" ]; then break; fi; ' \
+          'sleep 10; done; sleep 30'
+      ]
+    end
+
+    def artifacts
+      [
+        "curl -s --location --request GET \"https://#{qualys_opts[:gateway]}/csapi/v1.2/images/$image_id/software\" --header \"Authorization: Bearer $token\" | jq . > software.json", # rubocop:disable Layout/LineLength
+        "curl -s --location --request GET \"https://#{qualys_opts[:gateway]}/csapi/v1.2/images/$image_id/vuln\" --header \"Authorization: Bearer $token\" | jq . > vulnerabilities.json" # rubocop:disable Layout/LineLength
+      ]
+    end
+
+    def severities
+      [
+        "response=$(curl -s --location --request GET \"https://#{qualys_opts[:gateway]}/csapi/v1.2/images/$image_id/vuln/count\" --header \"Authorization: Bearer $token\")", # rubocop:disable Layout/LineLength
+        'severity5=$(jq -r ".severity5Count" <<< "${response}")',
+        'severity4=$(jq -r ".severity4Count" <<< "${response}")'
+      ]
+    end
+
+    def outputs # rubocop:disable Metrics/MethodLength
+      [
+        'if [ "$severity5" = "null" ]; then ' \
+        'echo "ERROR: Wrong ImageID or problem during vulnerabilities count." >&2; ' \
+        'exit 1; fi',
+        'if [ "$severity4" = "null" ]; then ' \
+        'echo "ERROR: Wrong ImageID or problem during vulnerabilities count." >&2; ' \
+        'exit 1; fi',
+        'echo "Severity5: $severity5, Severity4: $severity4"',
+        'risk=$((($severity5*3)+($severity4)))',
+        'echo "Risk: $risk"',
+        "if (($risk > \"#{qualys_opts[:acceptable_risk]}\")); then " \
+        'echo "Too many vulnerabilities. Severity5: $severity5, Severity4: $severity4" >&2; ' \
+        'exit 1; fi'
+      ]
+    end
+  end
+end

data/lib/pipedawg/skopeo_copy_job.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Pipedawg
+  # skopeo_copy_job class
+  class SkopeoCopyJob < Job
+    attr_accessor :skopeo_opts
+
+    def initialize(name = 'build', opts = {}, skopeo_opts = {})
+      @skopeo_opts = {
+        config: { '$CI_REGISTRY': { username: '$CI_REGISTRY_USER', password: '$CI_REGISTRY_PASSWORD' } },
+        copy_image: nil, debug: true, destinations: [{ copy_image: nil, flags: [], options: {} }],
+        flags: [], logins: {}, options: {}, skopeo: 'skopeo', stage: '${CI_PROJECT_DIR}/stage',
+        image: { entrypoint: [''], name: 'quay.io/skopeo/stable:latest' },
+        trusted_ca_cert_source_files: [], trusted_ca_cert_target_file: '/etc/docker/certs.d/ca.crt'
+      }.merge(skopeo_opts)
+      super name, opts
+      update
+    end
+
+    def update # rubocop:disable Metrics/AbcSize
+      require 'json'
+      opts[:image] = skopeo_opts[:image] if skopeo_opts[:image]
+      opts[:rules] = skopeo_opts[:rules] if skopeo_opts[:rules]
+      opts[:script] = debug + config + cert_copies + login + mkstage + pull + (
+        skopeo_opts[:destinations].map { |d| push(d) }
+      ).flatten(1)
+    end
+
+    private
+
+    def debug
+      if skopeo_opts[:debug]
+        Pipedawg::Util.echo_proxy_vars
+      else
+        []
+      end
+    end
+
+    def config
+      ['export CONFIG=$(mktemp -d)', "echo #{skopeo_opts[:config].to_json.inspect} > \"${CONFIG}/config.json\""]
+    end
+
+    def cert_copies
+      ["mkdir -p $(dirname \"#{skopeo_opts[:trusted_ca_cert_target_file]}\")"] +
+        Array(skopeo_opts[:trusted_ca_cert_source_files]).map do |cert|
+          "cat \"#{cert}\" >> \"#{skopeo_opts[:trusted_ca_cert_target_file]}\""
+        end
+    end
+
+    def login
+      skopeo_opts.fetch(:logins, {}).map do |k, v|
+        "echo \"#{v['password']}\" | #{skopeo_opts[:skopeo]} login --authfile \"${CONFIG}/config.json\" --username \"#{v['username']}\" --password-stdin \"#{k}\"" # rubocop:disable Layout/LineLength
+      end
+    end
+
+    def mkstage
+      ["mkdir -p \"#{skopeo_opts[:stage]}\""]
+    end
+
+    def pull
+      copy(skopeo_opts, "docker://#{skopeo_opts[:copy_image]}", "\"dir://#{skopeo_opts[:stage]}\"")
+    end
+
+    def push(destination_opts)
+      copy(destination_opts, "\"dir://#{skopeo_opts[:stage]}\"", "docker://#{destination_opts[:copy_image]}")
+    end
+
+    def copy(opts, source, destination)
+      Array(["#{skopeo_opts[:skopeo]} copy --authfile \"${CONFIG}/config.json\"", flags(opts), options(opts), source,
+             destination].reject(&:empty?).join(' '))
+    end
+
+    def flags(opts)
+      opts.fetch(:flags, []).uniq.map { |f| "--#{f}" }.join(' ')
+    end
+
+    def options(opts)
+      opts.fetch(:options, {}).map { |k, v| "--#{k} \"#{v}\"" }.join(' ')
+    end
+  end
+end

data/lib/pipedawg/util.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Pipedawg
+  # util class
+  class Util
+    def self.expand_env_vars(item) # rubocop:disable Metrics/MethodLength
+      case item
+      when Array
+        item.map { |i| expand_env_vars(i) }
+      when Hash
+        item.each { |k, v| item[k] = expand_env_vars(v) }
+        item.transform_keys! { |k| expand_env_vars(k) }
+        item
+      when String
+        item.gsub(/\${([^} ]+)}/) do |e|
+          ENV[e.gsub('${', '').gsub('}', '')]
+        end
+      else
+        item
+      end
+    end
+
+    def self.puts_proxy_vars
+      puts 'Proxy settings:'
+      puts "http_proxy: #{ENV['http_proxy']}"
+      puts "https_proxy: #{ENV['https_proxy']}"
+      puts "no_proxy: #{ENV['no_proxy']}"
+      puts "HTTP_PROXY: #{ENV['HTTP_PROXY']}"
+      puts "HTTPS_PROXY: #{ENV['HTTPS_PROXY']}"
+      puts "NO_PROXY: #{ENV['NO_PROXY']}"
+    end
+
+    def self.echo_proxy_vars
+      script = ['echo Proxy settings:']
+      script << 'echo   http_proxy: "${http_proxy}"'
+      script << 'echo   https_proxy: "${https_proxy}"'
+      script << 'echo   no_proxy: "${no_proxy}"'
+      script << 'echo   HTTP_PROXY: "${HTTP_PROXY}"'
+      script << 'echo   HTTPS_PROXY: "${HTTPS_PROXY}"'
+      script << 'echo   NO_PROXY: "${NO_PROXY}"'
+      script
+    end
+  end
+end

data/lib/pipedawg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pipedawg
-  VERSION = '0.3.1'
+  VERSION = '0.6.0'
 end

data/lib/pipedawg.rb

@@ -2,8 +2,11 @@
 
 require 'pipedawg/job'
 require 'pipedawg/helm_copy_job'
-require 'pipedawg/kaniko_job'
+require 'pipedawg/kaniko_build_job'
 require 'pipedawg/pipeline'
+require 'pipedawg/qualys_scan_job'
+require 'pipedawg/skopeo_copy_job'
+require 'pipedawg/util'
 require 'pipedawg/version'
 
 module Pipedawg

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pipedawg
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.6.0
 platform: ruby
 authors:
 - harbottle
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-02-08 00:00:00.000000000 Z
+date: 2022-02-24 00:00:00.000000000 Z
 dependencies: []
 description: Generate GitLab CI pipelines.
 email:
@@ -22,8 +22,11 @@ files:
 - lib/pipedawg.rb
 - lib/pipedawg/helm_copy_job.rb
 - lib/pipedawg/job.rb
-- lib/pipedawg/kaniko_job.rb
+- lib/pipedawg/kaniko_build_job.rb
 - lib/pipedawg/pipeline.rb
+- lib/pipedawg/qualys_scan_job.rb
+- lib/pipedawg/skopeo_copy_job.rb
+- lib/pipedawg/util.rb
 - lib/pipedawg/version.rb
 homepage: https://github.com/liger1978/pipedawg
 licenses: