pipedawg 0.5.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 53733705295ce1c9b2ae80fc1c8368e7e08e7d7a0e1fcf3727fe972dd5714fb1
-  data.tar.gz: 10e86751592488fbb9972605892d79af662e60e3a3da070292cf4933f31b7981
+  metadata.gz: 881d25335110dc811ee1570b6c3e909cee4bbf7e8209010d0287a159ee27d254
+  data.tar.gz: 0a1b664c0c074832b38a9ffb4700af77145ca1f102ca505d58eff2f558c24ca7
 SHA512:
-  metadata.gz: 11584d541a5d63f54761dee9ca878e3c79ba168ee1e5e93ec9ddccd559d869672a900a6aa25d966df4ef807a56ac0c75385644baf3bce5498ac7a68a29fd8a9a
-  data.tar.gz: 97f20d550f1cbe4bf298a2f6884f02754cd385e317d777887d11a8748d7436d8bb6db035944c6f59d2593b4e3a615db76f28c678bb392cf16ef3e6c6f92770a9
+  metadata.gz: b5c2156cfe2a3e73c31a974fc7bcc51144f28d1ba0a9ffef134d585dfd2663071682f84254f908e4df86b09e8ca8ae06a6bd422ebe4619ee554722a5eff72af4
+  data.tar.gz: 73058b94b1c44615aa5c5ede80dd010013021a7eab68b653019975464ed9b84b445f625897b4e08ee6982a0a537fee687171941316d5e8749117a767dc954116

data/README.md

@@ -32,7 +32,6 @@ Example:
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-# print_pipeline.rb
 require 'pipedawg'
 
 gem_job = Pipedawg::Job.new(
@@ -42,31 +41,32 @@ gem_job = Pipedawg::Job.new(
   script: ['bundle install', 'gem build *.gemspec']
 )
 
-kaniko_job = Pipedawg::KanikoJob.new(
+kaniko_build_job = Pipedawg::Job::Kaniko::Build.new(
   'build:kaniko',
-  {needs: ['build:gem'], retry: 2},
-  {context:'${CI_PROJECT_DIR}/docker',external_files: {'*.gem':'gems'}}
+  needs: ['build:gem'],
+  retry: 2,
+  context:'${CI_PROJECT_DIR}/docker',
+  external_files: {'*.gem':'gems'},
+  debug: false
 )
 
-pipeline = Pipedawg::Pipeline.new 'build:image', jobs: [gem_job, kaniko_job]
+pipeline = Pipedawg::Pipeline.new 'build:image', jobs: [gem_job, kaniko_build_job]
 puts pipeline.to_yaml
+pipeline.to_yaml_file('/tmp/pipeline.yaml')
 ```
 
 ```console
-$ chmod +x print_pipeline.rb
-$ ./print_pipeline.rb
+$ cat /tmp/pipeline.yaml 
 ---
 stages:
 - '1'
 - '2'
-workflow: {}
 build:gem:
   artifacts:
   - "*.gem"
   cache: {}
   image: ruby
   needs: []
-  rules: []
   script:
   - bundle install
   - gem build *.gemspec
@@ -75,16 +75,19 @@ build:gem:
 build:kaniko:
   artifacts: {}
   cache: {}
+  image:
+    entrypoint:
+    - ''
+    name: gcr.io/kaniko-project/executor:debug
   needs:
   - build:gem
   retry: 2
-  rules: []
   script:
   - echo "{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}"
     > "/kaniko/.docker/config.json"
   - cp "*.gem" "${CI_PROJECT_DIR}/docker/gems"
   - '"/kaniko/executor" --context "${CI_PROJECT_DIR}/docker" --dockerfile "Dockerfile"
-    --no-push'
+    --destination ${CI_REGISTRY_IMAGE}:latest'
   stage: '2'
   tags: []
 ```

data/lib/pipedawg/job/helm/copy.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Pipedawg
+  class Job
+    class Helm
+      # Pipedawg::Job::Helm::Copy class
+      class Copy < Job::Helm
+        def initialize(name, opts = {})
+          opts = {
+            chart: name,
+            destinations: [{ user: nil, password: nil, url: nil }],
+            password: nil, url: nil, user: nil, version: nil
+          }.merge(opts)
+          super name, opts
+          update
+        end
+
+        def update
+          opts[:script] = debug + pull + (opts[:destinations].map { |d| push(d) }).flatten(1)
+        end
+
+        private
+
+        def pull
+          case opts[:url]
+          when nil
+            []
+          when %r{^oci://}
+            pull_oci
+          else
+            pull_classic
+          end
+        end
+
+        def push(destination)
+          case destination[:url]
+          when nil
+            []
+          when %r{^oci://}
+            push_oci(destination)
+          else
+            push_classic(destination)
+          end
+        end
+
+        def pull_oci # rubocop:disable Metrics/AbcSize
+          script = []
+          if opts[:url] && opts[:chart] && opts[:version]
+            script = ['export HELM_EXPERIMENTAL_OCI=1']
+            script << login_oci(opts) if opts[:user] && opts[:password]
+            script << "\"#{opts[:command]}\" pull \"#{opts[:url]}/#{opts[:chart]}\" --version \"#{opts[:version]}\""
+          end
+          script
+        end
+
+        def push_oci(destination) # rubocop:disable Metrics/AbcSize
+          script = []
+          if destination[:url] && opts[:chart] && opts[:version]
+            script = ['export HELM_EXPERIMENTAL_OCI=1']
+            script << login_oci(destination) if destination[:user] && destination[:password]
+            script << "\"#{opts[:command]}\" push \"#{opts[:chart]}-#{opts[:version]}.tgz\" \"#{destination[:url]}\""
+          end
+          script
+        end
+
+        def login_oci(login_opts)
+          require 'uri'
+          "echo \"#{login_opts[:password]}\" | \"#{opts[:command]}\" registry login --username \"#{login_opts[:user]}\" --password-stdin \"#{URI(login_opts[:url]).host}\"" # rubocop:disable Layout/LineLength
+        end
+
+        def pull_classic # rubocop:disable Metrics/AbcSize
+          script = []
+          if opts[:url] && opts[:chart] && opts[:version]
+            suffix = login_classic(opts)
+            script << "\"#{opts[:command]}\" repo add source \"#{opts[:url]}\"#{suffix}"
+            script << "\"#{opts[:command]}\" repo update"
+            script << "\"#{opts[:command]}\" pull \"source/#{opts[:chart]}\" --version \"#{opts[:version]}\""
+          end
+          script
+        end
+
+        def push_classic(destination)
+          script = []
+          if destination[:url] && opts[:chart] && opts[:version]
+            script << plugin_classic
+            suffix = login_classic(destination)
+            script << "\"#{opts[:command]}\" cm-push \"#{opts[:chart]}-#{opts[:version]}.tgz\" \"#{destination[:url]}\"#{suffix}" # rubocop:disable Layout/LineLength
+          end
+          script
+        end
+
+        def login_classic(login_opts)
+          if login_opts[:user] && login_opts[:password]
+            " --username \"#{login_opts[:user]}\" --password \"#{login_opts[:password]}\""
+          else
+            ''
+          end
+        end
+
+        def plugin_classic
+          "\"#{opts[:command]}\" plugin list | grep -q cm-push || \"#{opts[:command]}\" plugin install https://github.com/chartmuseum/helm-push"
+        end
+      end
+    end
+  end
+end

data/lib/pipedawg/job/helm.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Pipedawg
+  class Job
+    # Pipedawg::Job::Helm class
+    class Helm < Job
+      def initialize(name, opts = {})
+        opts = {
+          command: 'helm',
+          image: { entrypoint: [''], name: 'alpine/helm' }
+        }.merge(opts)
+        super name, opts
+      end
+    end
+  end
+end

data/lib/pipedawg/job/kaniko/build.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Pipedawg
+  class Job
+    class Kaniko
+      # Pipedawg::Job::Kaniko::Build class
+      class Build < Job::Kaniko
+        def initialize(name, opts = {}) # rubocop:disable Metrics/MethodLength
+          opts = {
+            build_args: {},
+            config: { auths: { '$CI_REGISTRY': { username: '$CI_REGISTRY_USER', password: '$CI_REGISTRY_PASSWORD' } } },
+            config_file: '/kaniko/.docker/config.json', context: '${CI_PROJECT_DIR}',
+            destinations: ['${CI_REGISTRY_IMAGE}:latest'], dockerfile: 'Dockerfile', external_files: {}, flags: [],
+            ignore_paths: [], insecure_registries: [], options: {}, registry_certificates: {}, registry_mirrors: [],
+            skip_tls_verify_registry: [], trusted_ca_cert_source_files: [],
+            trusted_ca_cert_target_file: '/kaniko/ssl/certs/ca-certificates.crt'
+          }.merge(opts)
+          super name, opts
+          update
+        end
+
+        def update
+          require 'json'
+          opts[:script] = debug + config + cert_copies + file_copies + Array(kaniko_cmd)
+        end
+
+        private
+
+        def config
+          ["echo #{opts[:config].to_json.inspect} > \"#{opts[:config_file]}\""]
+        end
+
+        def cert_copies
+          Array(opts[:trusted_ca_cert_source_files]).map do |cert|
+            "cat \"#{cert}\" >> \"#{opts[:trusted_ca_cert_target_file]}\""
+          end
+        end
+
+        def file_copies
+          opts[:external_files].map do |source, dest|
+            "cp \"#{source}\" \"#{opts[:context]}/#{dest}\""
+          end
+        end
+
+        def kaniko_cmd # rubocop:disable Metrics/AbcSize
+          ["\"#{opts[:command]}\" --context \"#{opts[:context]}\"",
+           "--dockerfile \"#{opts[:dockerfile]}\"", flags, options, build_args,
+           ignore_paths, insecure_registries, registry_certificates, registry_mirrors,
+           destinations, skip_tls_verify_registries].reject(&:empty?).join(' ')
+        end
+
+        def flags
+          flags = opts[:flags].clone
+          flags << 'no-push' if opts[:destinations].empty?
+          flags.uniq.map { |f| "--#{f}" }.join(' ')
+        end
+
+        def options
+          opts[:options].map { |k, v| "--#{k}=\"#{v}\"" }.join(' ')
+        end
+
+        def build_args
+          opts[:build_args].map { |k, v| "--build-arg #{k}=\"#{v}\"" }.join(' ')
+        end
+
+        def ignore_paths
+          Array(opts[:ignore_paths]).map { |p| "--ignore-path #{p}" }.join(' ')
+        end
+
+        def insecure_registries
+          Array(opts[:insecure_registries]).map do |r|
+            "--insecure-registry #{r}"
+          end.join(' ')
+        end
+
+        def registry_certificates
+          opts[:registry_certificates].map do |k, v|
+            "--registry-certificate #{k}=\"#{v}\""
+          end.join(' ')
+        end
+
+        def registry_mirrors
+          Array(opts[:registry_mirrors]).map { |r| "--registry-mirror #{r}" }.join(' ')
+        end
+
+        def destinations
+          opts[:destinations].map { |d| "--destination #{d}" }.join(' ')
+        end
+
+        def skip_tls_verify_registries
+          Array(opts[:skip_tls_verify_registry]).map do |r|
+            "--skip-tls-verify-registry #{r}"
+          end.join(' ')
+        end
+      end
+    end
+  end
+end

data/lib/pipedawg/job/kaniko.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Pipedawg
+  class Job
+    # Pipedawg::Job::Kaniko class
+    class Kaniko < Job
+      def initialize(name, opts = {})
+        opts = {
+          command: '/kaniko/executor',
+          image: { entrypoint: [''], name: 'gcr.io/kaniko-project/executor:debug' }
+        }.merge(opts)
+        super name, opts
+      end
+    end
+  end
+end

data/lib/pipedawg/job/qualys/scan.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Pipedawg
+  class Job
+    class Qualys
+      # Pipedawg::Job::Qualys::Scan class
+      class Scan < Job::Qualys # rubocop:disable Metrics/ClassLength
+        def initialize(name, opts = {})
+          opts = {
+            acceptable_risk: '${QUALYS_ACCEPTABLE_IMAGE_RISK}',
+            artifacts: { expire_in: '1 month', paths: ['software.json', 'vulnerabilities.json'], when: 'always' },
+            config: { auths: { '$CI_REGISTRY': { username: '$CI_REGISTRY_USER', password: '$CI_REGISTRY_PASSWORD' } } },
+            gateway: '${QUALYS_GATEWAY}', image: nil, password: '${QUALYS_PASSWORD}',
+            scan_image: '${QUALYS_IMAGE}', scan_target_prefix: 'qualys_scan_target',
+            user: '${QUALYS_USERNAME}', variables: { GIT_STRATEGY: 'clone' }
+          }.merge(opts)
+          super name, opts
+          update
+        end
+
+        def update # rubocop:disable Metrics/AbcSize
+          require 'json'
+          opts[:script] =
+            debug + config + image + clean_config + token + scan_start +
+            scan_complete + artifacts + severities + outputs
+        end
+
+        private
+
+        def debug # rubocop:disable Metrics/MethodLength
+          if opts[:debug]
+            super + [
+              'echo Qualys settings:', "echo   Qualys gateway: \"#{opts[:gateway]}\"",
+              "echo   Qualys username: \"#{opts[:user]}\"",
+              "if [ \"#{opts[:password]}\" != '' ]; then " \
+              'echo   Qualys password is not empty; else ' \
+              'echo   Qualys password is not set; exit 1; fi'
+            ]
+          else
+            []
+          end
+        end
+
+        def config
+          ['export CONFIG=$(mktemp -d)', "echo #{opts[:config].to_json.inspect} > \"${CONFIG}/config.json\""]
+        end
+
+        def image
+          [
+            "image_target=\"#{opts[:scan_target_prefix]}:$(echo #{opts[:scan_image]} | sed 's/^[^/]*\\///'| sed 's/[:/]/-/g')\"", # rubocop:disable Layout/LineLength
+            "docker --config=\"${CONFIG}\" pull \"#{opts[:scan_image]}\"",
+            "docker image tag \"#{opts[:scan_image]}\" \"${image_target}\"",
+            "image_id=$(docker inspect --format=\"{{index .Id}}\" \"#{opts[:scan_image]}\" | cut -c8-19)",
+            'echo "Image ID: ${image_id}"'
+          ]
+        end
+
+        def clean_config
+          [
+            'rm -f "${CONFIG}/config.json"',
+            'rmdir "${CONFIG}"'
+          ]
+        end
+
+        def token
+          ["token=$(curl -s --location --request POST \"https://#{opts[:gateway]}/auth\" --header \"Content-Type: application/x-www-form-urlencoded\" --data-urlencode \"username=#{opts[:user]}\" --data-urlencode \"password=#{opts[:password]}\" --data-urlencode \"token=true\")"] # rubocop:disable Layout/LineLength
+        end
+
+        def scan_start
+          [
+            'while true; do ' \
+            "result=$(curl -s -o /dev/null -w ''%{http_code}'' --location --request GET \"https://#{opts[:gateway]}/csapi/v1.2/images/$image_id\" --header \"Authorization: Bearer $token\"); " + # rubocop:disable Layout/LineLength, Style/FormatStringToken
+              'echo "Waiting for scan to start..."; ' \
+              'echo "  Result: ${result}"; ' \
+              'if [ "${result}" = "200" ]; then break; fi; ' \
+              'sleep 10; done'
+          ]
+        end
+
+        def scan_complete
+          [
+            'while true; do ' \
+            "result=$(curl -s --location --request GET \"https://#{opts[:gateway]}/csapi/v1.2/images/$image_id\" --header \"Authorization: Bearer $token\" | jq -r '.scanStatus'); " + # rubocop:disable Layout/LineLength
+              'echo "Waiting for scan to complete..."; ' \
+              'echo "  Result: ${result}"; ' \
+              'if [ "${result}" = "SUCCESS" ]; then break; fi; ' \
+              'sleep 10; done; sleep 30'
+          ]
+        end
+
+        def artifacts
+          [
+            "curl -s --location --request GET \"https://#{opts[:gateway]}/csapi/v1.2/images/$image_id/software\" --header \"Authorization: Bearer $token\" | jq . > software.json", # rubocop:disable Layout/LineLength
+            "curl -s --location --request GET \"https://#{opts[:gateway]}/csapi/v1.2/images/$image_id/vuln\" --header \"Authorization: Bearer $token\" | jq . > vulnerabilities.json" # rubocop:disable Layout/LineLength
+          ]
+        end
+
+        def severities
+          [
+            "response=$(curl -s --location --request GET \"https://#{opts[:gateway]}/csapi/v1.2/images/$image_id/vuln/count\" --header \"Authorization: Bearer $token\")", # rubocop:disable Layout/LineLength
+            'severity5=$(jq -r ".severity5Count" <<< "${response}")',
+            'severity4=$(jq -r ".severity4Count" <<< "${response}")'
+          ]
+        end
+
+        def outputs # rubocop:disable Metrics/MethodLength
+          [
+            'if [ "$severity5" = "null" ]; then ' \
+            'echo "ERROR: Wrong ImageID or problem during vulnerabilities count." >&2; ' \
+            'exit 1; fi',
+            'if [ "$severity4" = "null" ]; then ' \
+            'echo "ERROR: Wrong ImageID or problem during vulnerabilities count." >&2; ' \
+            'exit 1; fi',
+            'echo "Severity5: $severity5, Severity4: $severity4"',
+            'risk=$((($severity5*3)+($severity4)))',
+            'echo "Risk: $risk"',
+            "if (($risk > \"#{opts[:acceptable_risk]}\")); then " \
+            'echo "Too many vulnerabilities. Severity5: $severity5, Severity4: $severity4" >&2; ' \
+            'exit 1; fi'
+          ]
+        end
+      end
+    end
+  end
+end

data/lib/pipedawg/job/qualys.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Pipedawg
+  class Job
+    # Pipedawg::Job::Qualys class
+    class Qualys < Job
+      def initialize(name, opts = {})
+        super name, opts
+      end
+    end
+  end
+end

data/lib/pipedawg/job/skopeo/copy.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Pipedawg
+  class Job
+    class Skopeo
+      # Pipedawg::Job::Skopeo::Copy class
+      class Copy < Job::Skopeo
+        def initialize(name, opts = {})
+          opts = {
+            config: {}, copy_image: name, destinations: [{ copy_image: nil, flags: [], options: {} }], flags: [],
+            logins: {}, options: {}, stage: '${CI_PROJECT_DIR}/stage', trusted_ca_cert_source_files: [],
+            trusted_ca_cert_target_file: '/etc/docker/certs.d/ca.crt'
+          }.merge(opts)
+          super name, opts
+          update
+        end
+
+        def update # rubocop:disable Metrics/AbcSize
+          require 'json'
+          opts[:script] = debug + config + cert_copies + login + mkstage + pull + (
+            opts[:destinations].map { |d| push(d) }
+          ).flatten(1)
+        end
+
+        private
+
+        def config
+          ['export CONFIG=$(mktemp -d)', "echo #{opts[:config].to_json.inspect} > \"${CONFIG}/config.json\""]
+        end
+
+        def cert_copies
+          ["mkdir -p $(dirname \"#{opts[:trusted_ca_cert_target_file]}\")"] +
+            Array(opts[:trusted_ca_cert_source_files]).map do |cert|
+              "cat \"#{cert}\" >> \"#{opts[:trusted_ca_cert_target_file]}\""
+            end
+        end
+
+        def login
+          opts.fetch(:logins, {}).map do |k, v|
+            "echo \"#{v['password']}\" | #{opts[:command]} login --authfile \"${CONFIG}/config.json\" --username \"#{v['username']}\" --password-stdin \"#{k}\"" # rubocop:disable Layout/LineLength
+          end
+        end
+
+        def mkstage
+          ["mkdir -p \"#{opts[:stage]}\""]
+        end
+
+        def pull
+          copy(opts, "docker://#{opts[:copy_image]}", "\"dir://#{opts[:stage]}\"")
+        end
+
+        def push(destination_opts)
+          copy(destination_opts, "\"dir://#{opts[:stage]}\"", "docker://#{destination_opts[:copy_image]}")
+        end
+
+        def copy(copy_opts, source, destination)
+          Array(["#{opts[:command]} copy --authfile \"${CONFIG}/config.json\"", flags(copy_opts), options(copy_opts),
+                 source, destination].reject(&:empty?).join(' '))
+        end
+
+        def flags(opts)
+          opts.fetch(:flags, []).uniq.map { |f| "--#{f}" }.join(' ')
+        end
+
+        def options(opts)
+          opts.fetch(:options, {}).map { |k, v| "--#{k} \"#{v}\"" }.join(' ')
+        end
+      end
+    end
+  end
+end

data/lib/pipedawg/job/skopeo.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Pipedawg
+  class Job
+    # Pipedawg::Job::Skopeo class
+    class Skopeo < Job
+      def initialize(name, opts = {})
+        opts = {
+          command: 'skopeo',
+          image: { entrypoint: [''], name: 'quay.io/skopeo/stable:latest' }
+        }.merge(opts)
+        super name, opts
+      end
+    end
+  end
+end

data/lib/pipedawg/job.rb

@@ -10,18 +10,31 @@ module Pipedawg
       @opts = {
         artifacts: {},
         cache: {},
+        debug: true,
         image: { name: 'ruby:2.5' },
         needs: [],
         retry: nil,
         rules: nil,
         script: [],
         stage: 'build',
-        tags: []
+        tags: [],
+        variables: nil
       }.merge(opts)
     end
 
     def to_hash
-      { "#{name}": opts.compact }
+      keys = %i[artifacts cache image needs retry rules script stage tags variables]
+      { "#{name}": opts.slice(*keys).compact }
+    end
+
+    private
+
+    def debug
+      if opts[:debug]
+        Pipedawg::Util.echo_proxy_vars
+      else
+        []
+      end
     end
   end
 end

data/lib/pipedawg/util.rb

@@ -9,6 +9,7 @@ module Pipedawg
         item.map { |i| expand_env_vars(i) }
       when Hash
         item.each { |k, v| item[k] = expand_env_vars(v) }
+        item.transform_keys! { |k| expand_env_vars(k) }
         item
       when String
         item.gsub(/\${([^} ]+)}/) do |e|

data/lib/pipedawg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pipedawg
-  VERSION = '0.5.0'
+  VERSION = '1.0.1'
 end

data/lib/pipedawg.rb

@@ -1,10 +1,15 @@
 # frozen_string_literal: true
 
 require 'pipedawg/job'
-require 'pipedawg/helm_copy_job'
-require 'pipedawg/kaniko_job'
+require 'pipedawg/job/helm'
+require 'pipedawg/job/helm/copy'
+require 'pipedawg/job/kaniko'
+require 'pipedawg/job/kaniko/build'
+require 'pipedawg/job/qualys'
+require 'pipedawg/job/qualys/scan'
+require 'pipedawg/job/skopeo'
+require 'pipedawg/job/skopeo/copy'
 require 'pipedawg/pipeline'
-require 'pipedawg/qualys_scan_job'
 require 'pipedawg/util'
 require 'pipedawg/version'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pipedawg
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 1.0.1
 platform: ruby
 authors:
 - harbottle
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-02-08 00:00:00.000000000 Z
+date: 2022-03-29 00:00:00.000000000 Z
 dependencies: []
 description: Generate GitLab CI pipelines.
 email:
@@ -20,11 +20,16 @@ files:
 - LICENSE.txt
 - README.md
 - lib/pipedawg.rb
-- lib/pipedawg/helm_copy_job.rb
 - lib/pipedawg/job.rb
-- lib/pipedawg/kaniko_job.rb
+- lib/pipedawg/job/helm.rb
+- lib/pipedawg/job/helm/copy.rb
+- lib/pipedawg/job/kaniko.rb
+- lib/pipedawg/job/kaniko/build.rb
+- lib/pipedawg/job/qualys.rb
+- lib/pipedawg/job/qualys/scan.rb
+- lib/pipedawg/job/skopeo.rb
+- lib/pipedawg/job/skopeo/copy.rb
 - lib/pipedawg/pipeline.rb
-- lib/pipedawg/qualys_scan_job.rb
 - lib/pipedawg/util.rb
 - lib/pipedawg/version.rb
 homepage: https://github.com/liger1978/pipedawg

data/lib/pipedawg/helm_copy_job.rb

@@ -1,115 +0,0 @@
-# frozen_string_literal: true
-
-module Pipedawg
-  # helm_copy_job class
-  class HelmCopyJob < Job
-    attr_accessor :helm_opts
-
-    def initialize(name = 'build', opts = {}, helm_opts = {})
-      @helm_opts = {
-        chart: name, debug: true,
-        destinations: [{ user: nil, password: nil, url: nil }],
-        helm: 'helm',
-        image: { entrypoint: [''], name: 'alpine/helm' },
-        password: nil, url: nil, user: nil, version: nil
-      }.merge(helm_opts)
-      super name, opts
-      update
-    end
-
-    def update # rubocop:disable Metrics/AbcSize
-      opts[:image] = helm_opts[:image] if helm_opts[:image]
-      opts[:script] = debug + pull + (helm_opts[:destinations].map { |d| push(d) }).flatten(1)
-    end
-
-    private
-
-    def debug
-      if helm_opts[:debug]
-        Pipedawg::Util.echo_proxy_vars
-      else
-        []
-      end
-    end
-
-    def pull
-      case helm_opts[:url]
-      when nil
-        []
-      when %r{^oci://}
-        pull_oci
-      else
-        pull_classic
-      end
-    end
-
-    def push(destination)
-      case destination[:url]
-      when nil
-        []
-      when %r{^oci://}
-        push_oci(destination)
-      else
-        push_classic(destination)
-      end
-    end
-
-    def pull_oci # rubocop:disable Metrics/AbcSize
-      script = []
-      if helm_opts[:url] && helm_opts[:chart] && helm_opts[:version]
-        script = ['export HELM_EXPERIMENTAL_OCI=1']
-        script << login_oci(helm_opts) if helm_opts[:user] && helm_opts[:password]
-        script << "\"#{helm_opts[:helm]}\" pull \"#{helm_opts[:url]}/#{helm_opts[:chart]}\" --version \"#{helm_opts[:version]}\"" # rubocop:disable Layout/LineLength
-      end
-      script
-    end
-
-    def push_oci(destination) # rubocop:disable Metrics/AbcSize
-      script = []
-      if destination[:url] && helm_opts[:chart] && helm_opts[:version]
-        script = ['export HELM_EXPERIMENTAL_OCI=1']
-        script << login_oci(destination) if destination[:user] && destination[:password]
-        script << "\"#{helm_opts[:helm]}\" push \"#{helm_opts[:chart]}-#{helm_opts[:version]}.tgz\" \"#{destination[:url]}\"" # rubocop:disable Layout/LineLength
-      end
-      script
-    end
-
-    def login_oci(login_opts)
-      require 'uri'
-      "echo \"#{login_opts[:password]}\" | \"#{helm_opts[:helm]}\" registry login --username \"#{login_opts[:user]}\" --password-stdin \"#{URI(login_opts[:url]).host}\"" # rubocop:disable Layout/LineLength
-    end
-
-    def pull_classic # rubocop:disable Metrics/AbcSize
-      script = []
-      if helm_opts[:url] && helm_opts[:chart] && helm_opts[:version]
-        suffix = login_classic(helm_opts)
-        script << "\"#{helm_opts[:helm]}\" repo add source \"#{helm_opts[:url]}\"#{suffix}"
-        script << "\"#{helm_opts[:helm]}\" repo update"
-        script << "\"#{helm_opts[:helm]}\" pull \"source/#{helm_opts[:chart]}\" --version \"#{helm_opts[:version]}\""
-      end
-      script
-    end
-
-    def push_classic(destination)
-      script = []
-      if destination[:url] && helm_opts[:chart] && helm_opts[:version]
-        script << plugin_classic
-        suffix = login_classic(destination)
-        script << "\"#{helm_opts[:helm]}\" cm-push \"#{helm_opts[:chart]}-#{helm_opts[:version]}.tgz\" \"#{destination[:url]}\"#{suffix}" # rubocop:disable Layout/LineLength
-      end
-      script
-    end
-
-    def login_classic(login_opts)
-      if login_opts[:user] && login_opts[:password]
-        " --username \"#{login_opts[:user]}\" --password \"#{login_opts[:password]}\""
-      else
-        ''
-      end
-    end
-
-    def plugin_classic
-      "\"#{helm_opts[:helm]}\" plugin list | grep -q cm-push || \"#{helm_opts[:helm]}\" plugin install https://github.com/chartmuseum/helm-push"
-    end
-  end
-end

data/lib/pipedawg/kaniko_job.rb

@@ -1,108 +0,0 @@
-# frozen_string_literal: true
-
-module Pipedawg
-  # kaniko_job class
-  class KanikoJob < Job
-    attr_accessor :kaniko_opts
-
-    def initialize(name = 'build', opts = {}, kaniko_opts = {}) # rubocop:disable Metrics/MethodLength
-      @kaniko_opts = {
-        build_args: {},
-        config: {
-          '$CI_REGISTRY': { username: '$CI_REGISTRY_USER', password: '$CI_REGISTRY_PASSWORD' }
-        },
-        config_file: '/kaniko/.docker/config.json', context: '${CI_PROJECT_DIR}', debug: true,
-        destinations: [], dockerfile: 'Dockerfile', executor: '/kaniko/executor', external_files: {},
-        flags: [], ignore_paths: [], insecure_registries: [],
-        image: { entrypoint: [''], name: 'gcr.io/kaniko-project/executor:debug' }, options: {},
-        registry_certificates: {}, registry_mirrors: [], skip_tls_verify_registry: [],
-        trusted_ca_cert_source_files: [], trusted_ca_cert_target_file: '/kaniko/ssl/certs/ca-certificates.crt'
-      }.merge(kaniko_opts)
-      super name, opts
-      update
-    end
-
-    def update # rubocop:disable Metrics/AbcSize
-      require 'json'
-      opts[:image] = kaniko_opts[:image] if kaniko_opts[:image]
-      opts[:script] = debug + config + cert_copies + file_copies + Array(kaniko_cmd)
-    end
-
-    private
-
-    def debug
-      if kaniko_opts[:debug]
-        Pipedawg::Util.echo_proxy_vars
-      else
-        []
-      end
-    end
-
-    def config
-      ["echo #{kaniko_opts[:config].to_json.inspect} > \"#{kaniko_opts[:config_file]}\""]
-    end
-
-    def cert_copies
-      Array(kaniko_opts[:trusted_ca_cert_source_files]).map do |cert|
-        "cat \"#{cert}\" >> \"#{kaniko_opts[:trusted_ca_cert_target_file]}\""
-      end
-    end
-
-    def file_copies
-      kaniko_opts[:external_files].map do |source, dest|
-        "cp \"#{source}\" \"#{kaniko_opts[:context]}/#{dest}\""
-      end
-    end
-
-    def kaniko_cmd # rubocop:disable Metrics/AbcSize
-      ["\"#{kaniko_opts[:executor]}\" --context \"#{kaniko_opts[:context]}\"",
-       "--dockerfile \"#{kaniko_opts[:dockerfile]}\"", flags, options, build_args,
-       ignore_paths, insecure_registries, registry_certificates, registry_mirrors,
-       destinations, skip_tls_verify_registries].reject(&:empty?).join(' ')
-    end
-
-    def flags
-      flags = kaniko_opts[:flags].clone
-      flags << 'no-push' if kaniko_opts[:destinations].empty?
-      flags.uniq.map { |f| "--#{f}" }.join(' ')
-    end
-
-    def options
-      kaniko_opts[:options].map { |k, v| "--#{k}=\"#{v}\"" }.join(' ')
-    end
-
-    def build_args
-      kaniko_opts[:build_args].map { |k, v| "--build-arg #{k}=\"#{v}\"" }.join(' ')
-    end
-
-    def ignore_paths
-      Array(kaniko_opts[:ignore_paths]).map { |p| "--ignore-path #{p}" }.join(' ')
-    end
-
-    def insecure_registries
-      Array(kaniko_opts[:insecure_registries]).map do |r|
-        "--insecure-registry #{r}"
-      end.join(' ')
-    end
-
-    def registry_certificates
-      kaniko_opts[:registry_certificates].map do |k, v|
-        "--registry-certificate #{k}=\"#{v}\""
-      end.join(' ')
-    end
-
-    def registry_mirrors
-      Array(kaniko_opts[:registry_mirrors]).map { |r| "--registry-mirror #{r}" }.join(' ')
-    end
-
-    def destinations
-      kaniko_opts[:destinations].map { |d| "--destination #{d}" }.join(' ')
-    end
-
-    def skip_tls_verify_registries
-      Array(kaniko_opts[:skip_tls_verify_registry]).map do |r|
-        "--skip-tls-verify-registry #{r}"
-      end.join(' ')
-    end
-  end
-end

data/lib/pipedawg/qualys_scan_job.rb

@@ -1,116 +0,0 @@
-# frozen_string_literal: true
-
-module Pipedawg
-  # qualys_scan_job class
-  class QualysScanJob < Job
-    attr_accessor :qualys_opts
-
-    def initialize(name = 'build', opts = {}, qualys_opts = {})
-      @qualys_opts = {
-        acceptable_risk: '${QUALYS_ACCEPTABLE_IMAGE_RISK}',
-        artifacts: {
-          expire_in: '1 month', paths: ['software.json', 'vulnerabilities.json'], when: 'always'
-        }, debug: true, gateway: '${QUALYS_GATEWAY}', image: nil, password: '${QUALYS_PASSWORD}', rules: nil,
-        scan_image: '${QUALYS_IMAGE}', scan_target_prefix: 'qualys_scan_target', tags: nil, user: '${QUALYS_USERNAME}',
-        variables: { GIT_STRATEGY: 'clone' }
-      }.merge(qualys_opts)
-      super name, opts
-      update
-    end
-
-    def update # rubocop:disable Metrics/AbcSize
-      require 'json'
-      opts[:artifacts] = qualys_opts[:artifacts] if qualys_opts[:artifacts]
-      opts[:image] = qualys_opts[:image]
-      opts[:rules] = qualys_opts[:rules] if qualys_opts[:rules]
-      opts[:tags] = qualys_opts[:tags] if qualys_opts[:tags]
-      opts[:variables] = qualys_opts[:variables] if qualys_opts[:variables]
-      opts[:script] = debug + image + token + scan_start + scan_complete + artifacts + severities + outputs
-    end
-
-    private
-
-    def debug # rubocop:disable Metrics/MethodLength
-      if qualys_opts[:debug]
-        Pipedawg::Util.echo_proxy_vars + [
-          'echo Qualys settings:',
-          "echo   Qualys gateway: \"#{qualys_opts[:gateway]}\"",
-          "echo   Qualys username: \"#{qualys_opts[:user]}\"",
-          "if [ \"#{qualys_opts[:password]}\" != '' ]; then " \
-          'echo   Qualys password is not empty; else ' \
-          'echo   Qualys password is not set; exit 1; fi'
-        ]
-      else
-        []
-      end
-    end
-
-    def image
-      [
-        "image_target=\"#{qualys_opts[:scan_target_prefix]}:$(echo #{qualys_opts[:scan_image]} | sed 's/^[^/]*\\///'| sed 's/[:/]/-/g')\"", # rubocop:disable Layout/LineLength
-        "docker pull \"#{qualys_opts[:scan_image]}\"",
-        "docker image tag \"#{qualys_opts[:scan_image]}\" \"${image_target}\"",
-        "image_id=$(docker inspect --format=\"{{index .Id}}\" \"#{qualys_opts[:scan_image]}\" | cut -c8-19)",
-        'echo "Image ID: ${image_id}"'
-      ]
-    end
-
-    def token
-      ["token=$(curl -s --location --request POST \"https://#{qualys_opts[:gateway]}/auth\" --header \"Content-Type: application/x-www-form-urlencoded\" --data-urlencode \"username=#{qualys_opts[:user]}\" --data-urlencode \"password=#{qualys_opts[:password]}\" --data-urlencode \"token=true\")"] # rubocop:disable Layout/LineLength
-    end
-
-    def scan_start
-      [
-        'while true; do ' \
-        "result=$(curl -s -o /dev/null -w ''%{http_code}'' --location --request GET \"https://#{qualys_opts[:gateway]}/csapi/v1.2/images/$image_id\" --header \"Authorization: Bearer $token\"); " + # rubocop:disable Layout/LineLength, Style/FormatStringToken
-          'echo "Waiting for scan to start..."; ' \
-          'echo "  Result: ${result}"; ' \
-          'if [ "${result}" = "200" ]; then break; fi; ' \
-          'sleep 10; done'
-      ]
-    end
-
-    def scan_complete
-      [
-        'while true; do ' \
-        "result=$(curl -s --location --request GET \"https://#{qualys_opts[:gateway]}/csapi/v1.2/images/$image_id\" --header \"Authorization: Bearer $token\" | jq -r '.scanStatus'); " + # rubocop:disable Layout/LineLength
-          'echo "Waiting for scan to complete..."; ' \
-          'echo "  Result: ${result}"; ' \
-          'if [ "${result}" = "SUCCESS" ]; then break; fi; ' \
-          'sleep 10; done; sleep 30'
-      ]
-    end
-
-    def artifacts
-      [
-        "curl -s --location --request GET \"https://#{qualys_opts[:gateway]}/csapi/v1.2/images/$image_id/software\" --header \"Authorization: Bearer $token\" | jq . > software.json", # rubocop:disable Layout/LineLength
-        "curl -s --location --request GET \"https://#{qualys_opts[:gateway]}/csapi/v1.2/images/$image_id/vuln\" --header \"Authorization: Bearer $token\" | jq . > vulnerabilities.json" # rubocop:disable Layout/LineLength
-      ]
-    end
-
-    def severities
-      [
-        "response=$(curl -s --location --request GET \"https://#{qualys_opts[:gateway]}/csapi/v1.2/images/$image_id/vuln/count\" --header \"Authorization: Bearer $token\")", # rubocop:disable Layout/LineLength
-        'severity5=$(jq -r ".severity5Count" <<< "${response}")',
-        'severity4=$(jq -r ".severity4Count" <<< "${response}")'
-      ]
-    end
-
-    def outputs # rubocop:disable Metrics/MethodLength
-      [
-        'if [ "$severity5" = "null" ]; then ' \
-        'echo "ERROR: Wrong ImageID or problem during vulnerabilities count." >&2; ' \
-        'exit 1; fi',
-        'if [ "$severity4" = "null" ]; then ' \
-        'echo "ERROR: Wrong ImageID or problem during vulnerabilities count." >&2; ' \
-        'exit 1; fi',
-        'echo "Severity5: $severity5, Severity4: $severity4"',
-        'risk=$((($severity5*3)+($severity4)))',
-        'echo "Risk: $risk"',
-        "if (($risk > \"#{qualys_opts[:acceptable_risk]}\")); then " \
-        'echo "Too many vulnerabilities. Severity5: $severity5, Severity4: $severity4" >&2; ' \
-        'exit 1; fi'
-      ]
-    end
-  end
-end