cyclid 0.2.5 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 075e82b45f23bff6cb7e99792eb97e3d7a55e912
-  data.tar.gz: 2ccbb21e53f9db060786e22d641a05203c18d420
+  metadata.gz: e7112455a4780d7e09049b9a7e248d41e70e4558
+  data.tar.gz: 5e7423b5c082b494a05d98ce46d87f1a72540111
 SHA512:
-  metadata.gz: e3bcc3a37f8c1bd4b83d90ed61af1bee3482d94b05b60c0966bb05f7fa433a618671d4450ab94890ffeb4fa5bd5b0c672af133067aae5dc68d3d7d0f01893501
-  data.tar.gz: 35cfcb20a3434bd9e0ff492c00017e64288b51cda48662417b61807e7cf4584f0cba83201592ad648f48a259c71776b55bbc3aff432f8321170dff0e134e4e7f
+  metadata.gz: 48fedf1d7150d25fa45787d1e47dc07568488f92151d3237440ecc69d6f71586edc4687eaad16d706758295905d4bda58b860e72422821e0df6462a994f40f92
+  data.tar.gz: a95452bf58313dec14d44e8c9c277c9e42c7d0222dd7ed453bd0f730121b2ee5fdf994c827811a4e2c731dc5e05aae84594466b029b523613e4012bf289e2d49

data/app/cyclid/controllers/users.rb

@@ -23,9 +23,9 @@ module Cyclid
     class UserController < ControllerBase
       helpers do
         # Remove sensitive data from the users data
-        def sanitize_user(user)
+        def sanitize_user(user, keys = %w(password secret))
           user.delete_if do |key, _value|
-            key == 'password' || key == 'secret'
+            keys.include? key
           end
         end
       end

data/app/cyclid/controllers/users/document.rb

@@ -78,7 +78,8 @@ module Cyclid
             user_hash = user.serializable_hash
             user_hash['organizations'] = user.organizations.map(&:name)
 
-            user_hash = sanitize_user(user_hash)
+            # DO provide the users HMAC secret, in this instance
+            user_hash = sanitize_user(user_hash, ['password'])
 
             return user_hash.to_json
           end

data/app/cyclid/job/evaluator.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+# Copyright 2017 Liqwyd Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Top level module for the core Cyclid code.
+module Cyclid
+  # Module for the Cyclid API
+  module API
+    # Module for Cyclid Job related classes
+    module Job
+      # Evaluator exception class
+      class EvalException < RuntimeError
+      end
+
+      # Evalute an expression for "only_if" & "not_if"
+      class Evaluator
+        class << self
+          def only_if(statement, vars)
+            evaluate(statement, vars)
+          end
+
+          def not_if(statement, vars)
+            not evaluate(statement, vars) # rubocop:disable Style/Not
+          end
+
+          private
+
+          def compare(lvalue, operator, rvalue)
+            case operator
+            # Loose (case insensitive) comparision
+            when '=='
+              lvalue.downcase == rvalue.downcase # rubocop:disable Performance/Casecmp
+            # Case sensitive/value comparision
+            when '===', 'eq'
+              lvalue == rvalue
+            # Not-equal
+            when '!=', 'ne'
+              lvalue != rvalue
+            # Less than
+            when '<', 'lt'
+              lvalue.to_f < rvalue.to_f
+            # Greater than
+            when '>', 'gt'
+              lvalue.to_f > rvalue.to_f
+            # Less than or equal
+            when '<=', 'le'
+              lvalue.to_f <= rvalue.to_f
+            # Greater than or equal
+            when '>=', 'ge'
+              lvalue.to_f >= rvalue.to_f
+            # Not an operator we know
+            else
+              raise EvalException, "unknown operator: #{operator}"
+            end
+          end
+
+          def evaluate(statement, vars)
+            # Replace single % characters with escaped versions and interpolate
+            expr = statement.gsub(/%([^{])/, '%%\1') ** vars
+
+            # Evaluate for both string comparisons:
+            #
+            # 'string1' == 'string2'
+            #
+            # and numbers:
+            #
+            # 1 < 2
+            #
+            # Numbers can be integers, floats or percentages
+            #
+            # Only the ==, != and === operators are recognised for strings. All
+            # operators, including == & != are valid for Numbers: === is not a valid
+            # operator for numbers.
+
+            # rubocop:disable Metrics/LineLength
+            case expr
+            when /\A'(.*)' (==|!=|===) '(.*)'\Z/,
+                 /\A([0-9]*|[0-9]*\.[0-9]*)%? (==|eq|!=|ne|<|lt|>|gt|<=|le|>=|ge) ([0-9]*|[0-9]*\.[0-9]*)%?\Z/
+              compare(Regexp.last_match[1],
+                      Regexp.last_match[2],
+                      Regexp.last_match[3])
+            else
+              raise EvalException, "unable to evaluate #{expr}"
+            end
+            # rubocop:enable Metrics/LineLength
+          end
+        end
+      end
+    end
+  end
+end

data/app/cyclid/job/job.rb

@@ -134,14 +134,27 @@ module Cyclid
             job_sequence << stage_success \
               unless job_stage[:on_success].nil? or \
                      stage?(job_sequence, job_stage[:on_success])
-            stage_view.on_success = job_stage[:on_success]
 
+            # Set the on_success handler; if no explicit hander is defined, use
+            # the next stage in the sequence
+            success_stage = if job_stage[:on_success]
+                              job_stage[:on_success]
+                            else
+                              next_stage(job_sequence, job_stage)
+                            end
+            stage_view.on_success = success_stage
+
+            # Now set the on_failure failure
             stage_failure = { stage: job_stage[:on_failure] }
             job_sequence << stage_failure \
               unless job_stage[:on_failure].nil? or \
                      stage?(job_sequence, job_stage[:on_failure])
             stage_view.on_failure = job_stage[:on_failure]
 
+            # Merge in any modifiers
+            stage_view.only_if = job_stage[:only_if]
+            stage_view.not_if = job_stage[:not_if]
+
             # Store the modified StageView
             stages[stage_view.name.to_sym] = stage_view
           end
@@ -158,6 +171,14 @@ module Cyclid
           end
           return found
         end
+
+        # Get the directly proceeding stage in the sequence
+        def next_stage(sequence, stage)
+          idx = sequence.index stage
+
+          next_stage = sequence.at(idx + 1)
+          next_stage.nil? ? nil : next_stage['stage']
+        end
       end
     end
   end

data/app/cyclid/job/runner.rb

@@ -124,13 +124,37 @@ module Cyclid
             stage_definition = stages[sequence.to_sym]
             stage = Oj.load(stage_definition, symbol_keys: true)
 
-            @notifier.write "#{'-' * 79}\n#{Time.now} : " \
-                            "Running stage #{stage.name} v#{stage.version}\n"
+            # Evaluate any only_if/not_if expressions. Always run the stage if there are no
+            # modifiers.
+            do_run = if stage.only_if
+                       Evaluator.only_if(stage.only_if, @ctx)
+                     elsif stage.not_if
+                       Evaluator.not_if(stage.not_if, @ctx)
+                     else
+                       true
+                     end
+
+            if do_run
+              @notifier.write "#{'-' * 79}\n#{Time.now} : " \
+                              "Running stage #{stage.name} v#{stage.version}\n"
+
+              # Run the stage
+              success, rc = run_stage(stage)
+
+              Cyclid.logger.info "stage #{(success ? 'succeeded' : 'failed')} and returned #{rc}"
+            else
+              @notifier.write "#{'-' * 79}\n#{Time.now} : " \
+                              "Skipping stage #{stage.name} v#{stage.version}\n"
 
-            # Run the stage
-            success, rc = run_stage(stage)
+              # Skip this stage; assume success
+              success = true
+              rc = 0
 
-            Cyclid.logger.info "stage #{(success ? 'succeeded' : 'failed')} and returned #{rc}"
+              # rubocop:disable Style/MultilineTernaryOperator
+              Cyclid.logger.info "stage skipped due to #{stage.only_if ? \
+                                 "only_if #{stage.only_if}" : "not_if #{stage.not_if}"}"
+              # rubocop:enable Style/MultilineTernaryOperator
+            end
 
             # Decide which stage to run next depending on the outcome of this
             # one

data/app/cyclid/job/stage.rb

@@ -25,7 +25,7 @@ module Cyclid
       # the database object.
       class StageView
         attr_reader :name, :version, :steps
-        attr_accessor :on_success, :on_failure
+        attr_accessor :on_success, :on_failure, :only_if, :not_if
 
         def initialize(arg)
           if arg.is_a? Cyclid::API::Stage

data/app/cyclid/monkey_patches.rb

@@ -9,10 +9,10 @@ class Hash
   end
 
   # Interpolate the data in the ctx hash into any String values
-  def interpolate(ctx)
+  def %(other)
     hmap do |key, value|
       if value.is_a? String
-        { key => value % ctx }
+        { key => value ** other }
       else
         { key => value }
       end
@@ -23,15 +23,42 @@ end
 # Add a method to Array
 class Array
   # Interpolate the data in the ctx hash for each String & Hash item
-  def interpolate(ctx)
+  def %(other)
     map do |entry|
       if entry.is_a? Hash
-        entry.interpolate ctx
+        entry % other
       elsif entry.is_a? String
-        entry % @ctx
+        entry ** other
       else
         entry
       end
     end
   end
 end
+
+# Add a method to String
+class String
+  # Provide a "safe" version of the % (interpolation) operator; if a key does
+  # not exist in arg, catch the KeyError and insert it as a nil value, and
+  # continue.
+  #
+  # We're using the ** operator as it's one that isn't already used by String,
+  # and it's abusing % already so hey, why not. FTP
+  def **(other)
+    res = nil
+    arg = other ? other.dup : {}
+
+    begin
+      res = self % arg
+    rescue KeyError => ex
+      # Extract the key name from the exception message (sigh)
+      match = ex.message.match(/\Akey{(.*)} not found\Z/)
+      key = match[1]
+
+      # Inject key with a default value and try again
+      arg[key.to_sym] = nil
+    end while res.nil? # rubocop:disable Lint/Loop
+
+    return res
+  end
+end

data/app/cyclid/plugins/action/command.rb

@@ -55,7 +55,7 @@ module Cyclid
         def perform(log)
           begin
             # Export the environment data to the build host, if necesary
-            env = @env.interpolate(@ctx) if @env
+            env = @env % @ctx if @env
             @transport.export_env(env)
 
             # Log the command being run (and the working directory, if one is
@@ -64,11 +64,11 @@ module Cyclid
             log.write(@path.nil? ? "$ #{cmd_args}\n" : "$ #{@path} : #{cmd_args}\n")
 
             # Interpolate any data from the job context
-            cmd_args = cmd_args % @ctx
+            cmd_args = cmd_args ** @ctx
 
             # Interpolate the path if one is set
             path = @path
-            path = path % @ctx unless path.nil?
+            path = path ** @ctx unless path.nil?
 
             # Run the command
             success = @transport.exec(cmd_args, path)

data/app/cyclid/plugins/action/email.rb

@@ -51,9 +51,9 @@ module Cyclid
                                 "as #{email_config[:from]}"
 
             # Add the job context
-            to = @to % @ctx
-            subject = @subject % @ctx
-            message = @message % @ctx
+            to = @to ** @ctx
+            subject = @subject ** @ctx
+            message = @message ** @ctx
 
             # Create a binding for the text & HTML ERB templates
             info = { color: @color, title: subject }

data/app/cyclid/plugins/action/log.rb

@@ -32,7 +32,7 @@ module Cyclid
 
         # Write the log message, with the context data interpolated
         def perform(log)
-          log.write(@message % @ctx)
+          log.write("#{@message ** @ctx}\n")
           true
         end
 

data/app/cyclid/plugins/action/script.rb

@@ -56,12 +56,12 @@ module Cyclid
         def perform(log)
           begin
             # Export the environment data to the build host, if necesary
-            env = @env.interpolate(@ctx) if @env
+            env = @env % @ctx if @env
             @transport.export_env(env)
 
             # Add context data
-            script = @script % @ctx
-            path = @path % @ctx
+            script = @script ** @ctx
+            path = @path ** @ctx
 
             # Create an IO object containing the script and upload it to the
             # build host

data/app/cyclid/plugins/action/slack.rb

@@ -43,15 +43,15 @@ module Cyclid
             Cyclid.logger.debug "using plugin config #{plugin_data}"
             config = plugin_data['config']
 
-            subject = @subject % @ctx
+            subject = @subject ** @ctx
 
             url = @url || config['webhook_url']
             raise 'no webhook URL given' if url.nil?
 
-            url = url % @ctx
+            url = url ** @ctx
             Cyclid.logger.debug "sending notification to #{url}"
 
-            message_text = @message % @ctx if @message
+            message_text = @message ** @ctx if @message
 
             # Create a binding for the template
             bind = binding

data/app/cyclid/plugins/api/github/helpers.rb

@@ -33,8 +33,8 @@ module Cyclid
               @pr_head ||= pull_request['head']
             end
 
-            def pr_clone_url
-              pr_head['repo']['html_url']
+            def pr_base
+              @pr_base ||= pull_request['base']
             end
 
             def pr_sha
@@ -45,22 +45,29 @@ module Cyclid
               pr_head['ref']
             end
 
-            def pr_repo
-              @pr_repo ||= pr_head['repo']
+            def pr_base_repo
+              @pr_base_repo ||= pr_base['repo']
             end
 
-            def pr_status_url
-              url = pr_repo['statuses_url']
-              @pr_status_url ||= url.gsub('{sha}', pr_sha)
+            def pr_base_url
+              pr_base_repo['html_url']
+            end
+
+            def pr_head_repo
+              @pr_head_repo ||= pr_head['repo']
+            end
+
+            def pr_head_url
+              pr_head_repo['html_url']
             end
 
             def pr_trees_url
-              url = pr_repo['trees_url']
+              url = pr_head_repo['trees_url']
               @pr_trees_url ||= url.gsub('{/sha}', "/#{pr_sha}")
             end
 
             def pr_repository
-              @repo ||= Octokit::Repository.from_url(pr_clone_url)
+              @repo ||= Octokit::Repository.from_url(pr_base_url)
             end
 
             def push_head_commit
@@ -131,6 +138,23 @@ module Cyclid
               state = "#{org.name}:#{org.salt}:#{org.owner_email}"
               Base64.urlsafe_encode64(Digest::SHA256.digest(state))
             end
+
+            # Normalize a Github URL into a Cyclid style source definition
+            def normalize(url)
+              uri = URI.parse(url)
+
+              source = {}
+              source['url'] = "#{uri.scheme}://#{uri.host}#{uri.path.gsub(/.git$/, '')}"
+              source['token'] = uri.user if uri.user
+              source['branch'] = uri.fragment if uri.fragment
+
+              source
+            end
+
+            # Extract the "humanish" part from a Git repository URL
+            def humanish(uri)
+              uri.path.split('/').last.gsub('.git', '')
+            end
           end
         end
       end

data/app/cyclid/plugins/api/github/pull_request.rb

@@ -43,7 +43,7 @@ module Cyclid
 
               # Get the list of files in the root of the repository in the
               # Pull Request branch
-              clone_url = URI(pr_clone_url)
+              clone_url = URI(pr_head_url)
 
               # Get the authentication key
               auth_token = find_oauth_token(config, clone_url)
@@ -78,15 +78,19 @@ module Cyclid
                 job_definition = load_job_file(pr_repository, job_sha, job_type)
 
                 # Insert this repository & branch into the sources
+                clone_source = normalize(clone_url.to_s)
+                clone_source['type'] = 'git'
+                clone_source['branch'] = pr_ref
+                clone_source['token'] = auth_token
+
+                # We need to avoid causing a collision between the PR source
+                # (which may be from a forked repo) and any source definitions
+                # which may exist in the job file (which may be the "base"
+                # repository.
                 #
-                # XXX Could this cause collisions between the existing sources in
-                # the job definition? Not entirely sure what the workflow will
-                # look like.
-                job_sources = job_definition['sources'] || []
-                job_sources << { 'type' => 'git',
-                                 'url' => clone_url.to_s,
-                                 'branch' => pr_ref,
-                                 'token' => auth_token }
+                # Compare everything and try to match any duplicates, and
+                # flatten the sources.
+                job_sources = insert_or_update_source(job_definition['sources'] || [], clone_source)
                 job_definition['sources'] = job_sources
 
                 Cyclid.logger.debug "sources=#{job_definition['sources']}"
@@ -110,10 +114,10 @@ module Cyclid
                 linkback_url = "#{ui_url}/#{organization_name}"
 
                 # Inject some useful context data
-                ctx = { gh_event: 'pull_request',
-                        gh_user: pull_request['user']['login'],
-                        gh_ref: pr_ref,
-                        gh_comment: pull_request['body'] }
+                ctx = { github_event: 'pull_request',
+                        github_user: pull_request['user']['login'],
+                        github_ref: pr_ref,
+                        github_comment: pull_request['title'] }
 
                 callback = GithubCallback.new(auth_token, pr_repository, pr_sha, linkback_url)
                 job_from_definition(job_definition, callback, ctx)
@@ -126,6 +130,34 @@ module Cyclid
 
               return true
             end
+
+            # Either insert (append) the Pull Request head repository, or
+            # replace an existing definition; for example if the job contains
+            # a source definition for "https://github.com/foo/bar" and the PR
+            # head is "https://github.com/baz/bar", replace "foo/bar" with
+            # "baz/bar"
+            def insert_or_update_source(sources, new_source)
+              updated = false
+              new_uri = URI.parse(new_source['url'])
+
+              normalized = sources.map do |source|
+                uri = URI.parse(source['url'])
+                next unless uri.scheme =~ /\Ahttps?\z/
+
+                # If the "humanish" components match, use the new definition.
+                if humanish(uri) == humanish(new_uri)
+                  updated = true
+                  new_source
+                else
+                  source
+                end
+              end
+
+              # If we didn't update an existing source definition, insert the new one
+              normalized << new unless updated
+
+              normalized.compact
+            end
           end
         end
       end

data/app/cyclid/plugins/api/github/push.rb

@@ -97,10 +97,10 @@ module Cyclid
                 linkback_url = "#{ui_url}/#{organization_name}"
 
                 # Inject some useful context data
-                ctx = { gh_event: 'push',
-                        gh_user: @payload['sender']['login'],
-                        gh_ref: push_ref,
-                        gh_comment: push_head_commit['message'] }
+                ctx = { github_event: 'push',
+                        github_user: @payload['sender']['login'],
+                        github_ref: push_ref,
+                        github_comment: push_head_commit['message'] }
 
                 callback = GithubCallback.new(auth_token, push_repository, push_sha, linkback_url)
                 job_from_definition(job_definition, callback, ctx)

data/app/cyclid/plugins/provisioner/debian.rb

@@ -26,7 +26,7 @@ module Cyclid
           transport.export_env('DEBIAN_FRONTEND' => 'noninteractive')
 
           # Build hosts may require an update before anything can be installed
-          success = transport.exec 'apt-get update'
+          success = transport.exec 'apt-get update -qq'
           raise 'failed to update repositories' unless success
 
           if env.key? :repos
@@ -44,13 +44,13 @@ module Cyclid
             end
 
             # We must update again to cache the new repositories
-            success = transport.exec 'apt-get update'
+            success = transport.exec 'apt-get update -q'
             raise 'failed to update repositories' unless success
           end
 
           if env.key? :packages
             success = transport.exec \
-              "apt-get install -y #{env[:packages].join(' ')}" \
+              "apt-get install -q -y #{env[:packages].join(' ')}" \
 
             raise "failed to install packages #{env[:packages].join(' ')}" unless success
           end

data/app/cyclid/plugins/provisioner/ubuntu.rb

@@ -26,12 +26,12 @@ module Cyclid
           transport.export_env('DEBIAN_FRONTEND' => 'noninteractive')
 
           # Build hosts may require an update before anything can be installed
-          success = transport.exec 'apt-get update'
+          success = transport.exec 'apt-get update -qq'
           raise 'failed to update repositories' unless success
 
           if env.key? :repos
-            # Ensure apt-get-repository is available
-            transport.exec 'apt-get install -y software-properties-common'
+            # Ensure apt-add-repository is available
+            transport.exec 'apt-get install -qq -y software-properties-common'
 
             env[:repos].each do |repo|
               next unless repo.key? :url
@@ -50,13 +50,13 @@ module Cyclid
             end
 
             # We must update again to cache the new repositories
-            success = transport.exec 'apt-get update'
+            success = transport.exec 'apt-get update -q'
             raise 'failed to update repositories' unless success
           end
 
           if env.key? :packages
             success = transport.exec \
-              "apt-get install -y #{env[:packages].join(' ')}" \
+              "apt-get install -q -y #{env[:packages].join(' ')}" \
 
             raise "failed to install packages #{env[:packages].join(' ')}" unless success
           end

data/app/cyclid/plugins/source/git.rb

@@ -32,7 +32,7 @@ module Cyclid
               unless source.key? :url
 
             # Add any context data (which could include secrets)
-            source = source.interpolate(ctx)
+            source = source % ctx
 
             url = URI(source[:url])
 

data/lib/cyclid/version.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 module Cyclid
   module Api
-    VERSION = '0.2.5'
+    VERSION = '0.3.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cyclid
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.3.0
 platform: ruby
 authors:
 - Kristian Van Der Vliet
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-10 00:00:00.000000000 Z
+date: 2017-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: oj
@@ -374,6 +374,7 @@ files:
 - app/cyclid/controllers/users/document.rb
 - app/cyclid/health_helpers.rb
 - app/cyclid/job.rb
+- app/cyclid/job/evaluator.rb
 - app/cyclid/job/helpers.rb
 - app/cyclid/job/job.rb
 - app/cyclid/job/runner.rb