cyclid 0.2.0

data/app/cyclid/controllers/users/document.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+# Copyright 2016 Liqwyd Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Top level module for all of the core Cyclid code.
+module Cyclid
+  # Module for the Cyclid API
+  module API
+    # Module for all User related API endpoints
+    module Users
+      # API endpoints for a single Organization document
+      # @api REST
+      module Document
+        # @!group Users
+
+        # @!method get_users_user
+        # @overload GET /users/:username
+        # @macro rest
+        # @param [String] username Username of the user.
+        # Get a specific user.
+        # @return The requested user.
+        # @return [404] The user does not exist
+
+        # @!method put_users_user(body)
+        # @overload PUT /users/:username
+        # @macro rest
+        # @param [String] username Username of the user.
+        # Modify a specific user.
+        # @param [JSON] body User information
+        # @option body [String] name Users real name
+        # @option body [String] email Users new email address
+        # @option body [String] password New Bcrypt2 encrypted password
+        # @option body [String] new_password New password in plain text, which will be
+        #   encrypted before being stored in the databaase.
+        # @option body [String] secret New HMAC signing secret. This should be a suitably
+        #   long random string.
+        # @return [200] User was modified successfully
+        # @return [400] The user definition is invalid
+        # @return [404] The user does not exist
+
+        # @!method delete_users_user
+        # @overload DELETE /users/:username
+        # @macro rest
+        # @param [String] username Username of the user.
+        # Delete a specific user.
+        # @return [200] User was deleted successfully
+        # @return [404] The user does not exist
+
+        # @!endgroup
+
+        # Sinatra callback
+        # @private
+        def self.registered(app)
+          include Errors::HTTPErrors
+
+          # Get a specific user.
+          app.get do
+            authorized_as!(params[:username], Operations::READ)
+
+            user = User.find_by(username: params[:username])
+            halt_with_json_response(404, INVALID_USER, 'user does not exist') \
+              if user.nil?
+
+            Cyclid.logger.debug user.organizations
+
+            # Convert to a Hash and inject the User data
+            user_hash = user.serializable_hash
+            user_hash['organizations'] = user.organizations.map(&:name)
+
+            user_hash = sanitize_user(user_hash)
+
+            return user_hash.to_json
+          end
+
+          # Modify a specific user.
+          app.put do
+            authorized_as!(params[:username], Operations::WRITE)
+
+            payload = parse_request_body
+            Cyclid.logger.debug payload
+
+            user = User.find_by(username: params[:username])
+            halt_with_json_response(404, INVALID_USER, 'user does not exist') \
+              if user.nil?
+
+            begin
+              user.name = payload['name'] if payload.key? 'name'
+              user.email = payload['email'] if payload.key? 'email'
+              user.password = payload['password'] if payload.key? 'password'
+              user.secret = payload['secret'] if payload.key? 'secret'
+              user.new_password = payload['new_password'] if payload.key? 'new_password'
+              user.save!
+            rescue ActiveRecord::ActiveRecordError => ex
+              Cyclid.logger.debug ex.message
+              halt_with_json_response(400, INVALID_JSON, ex.message)
+            end
+
+            return json_response(NO_ERROR, "user #{payload['username']} modified")
+          end
+
+          # Delete a specific user.
+          app.delete do
+            authorized_as!(params[:username], Operations::ADMIN)
+
+            user = User.find_by(username: params[:username])
+            halt_with_json_response(404, INVALID_USER, 'user does not exist') \
+              if user.nil?
+
+            begin
+              user.delete
+            rescue ActiveRecord::ActiveRecordError => ex
+              Cyclid.logger.debug ex.message
+              halt_with_json_response(400, INVALID_JSON, ex.message)
+            end
+
+            return json_response(NO_ERROR, "user #{params['username']} deleted")
+          end
+        end
+      end
+    end
+  end
+end

data/app/cyclid/health_helpers.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+# Copyright 2016 Liqwyd Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'sinatra-health-check'
+
+# Top level module for all of the core Cyclid code.
+module Cyclid
+  # Module for the Cyclid API
+  module API
+    module Health
+      # Helper methods to isolate the plugins from the implementation details
+      # of the healthcheck framework
+      module Helpers
+        # Health statuses
+        STATUSES = {
+          ok: SinatraHealthCheck::Status::SEVERITIES[:ok],
+          warning: SinatraHealthCheck::Status::SEVERITIES[:warning],
+          error: SinatraHealthCheck::Status::SEVERITIES[:error]
+        }.freeze
+
+        # Produce a SinatraHealthCheck object from the given status & message
+        def health_status(status, message)
+          SinatraHealthCheck::Status.new(status, message)
+        end
+      end
+    end
+  end
+end

data/app/cyclid/job.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+# Load all of the job related classes
+require_rel 'job/*.rb'

data/app/cyclid/job/helpers.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+# Copyright 2016 Liqwyd Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Top level module for the core Cyclid code.
+module Cyclid
+  # Module for the Cyclid API
+  module API
+    # Module for Cyclid Job related classes
+    module Job
+      # Useful methods for dealing with Jobs
+      module Helpers
+        # Create & dispatch a Job from the job definition
+        def job_from_definition(definition, callback = nil, context = {})
+          # This function will only ever be called from a Sinatra context
+          org = Organization.find_by(name: params[:name])
+          halt_with_json_response(404, INVALID_ORG, 'organization does not exist') \
+            if org.nil?
+
+          # Create a new JobRecord
+          job_record = JobRecord.new
+          job_record.started = Time.now.to_s
+          job_record.status = Constants::JobStatus::NEW
+          job_record.save!
+
+          org.job_records << job_record
+
+          # The user may, or may not, be set: if the job has come via. the :organization/jobs
+          # endpoint it'll be set (as that's authenticated), if it's come from an API extension the
+          # user mat not be set (as it may be unauthenticated, or not using the same authentication
+          # as Cyclid)
+          user = current_user
+          current_user.job_records << job_record if user
+
+          begin
+            job = ::Cyclid::API::Job::JobView.new(definition, context, org)
+            Cyclid.logger.debug job.to_hash
+
+            job_id = Cyclid.dispatcher.dispatch(job, job_record, callback)
+          rescue StandardError => ex
+            Cyclid.logger.error "job dispatch failed: #{ex}"
+
+            # We couldn't dispatch the job; record the failure
+            job_record.status = Constants::JobStatus::FAILED
+            job_record.ended = Time.now.to_s
+            job_record.save!
+
+            raise
+          end
+
+          return job_id
+        end
+      end
+    end
+  end
+end

data/app/cyclid/job/job.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+# Copyright 2016 Liqwyd Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Top level module for the core Cyclid code.
+module Cyclid
+  # Module for the Cyclid API
+  module API
+    # Module for Cyclid Job related classes
+    module Job
+      # Non-ActiveRecord class which holds a complete Job, complete with
+      # serialised stages and the resolved sequence.
+      class JobView
+        attr_reader :name, :version
+
+        def initialize(job, context, org)
+          # Job is a hash (converted from JSON or YAML)
+          job.symbolize_keys!
+
+          @name = job[:name]
+          @version = job[:version] || '1.0.0'
+
+          @context = context
+          @organization = org.name
+          @environment = job[:environment]
+          @sources = job[:sources] || []
+          @secrets = setec_astronomy(org, (job[:secrets] || {}))
+
+          # Build a single unified list of StageViews
+          @stages, @sequence = build_stage_collection(job, org)
+        end
+
+        # Return everything, serialized into a hash
+        def to_hash
+          hash = {}
+          hash[:name] = @name
+          hash[:version] = @version
+          hash[:context] = @context
+          hash[:organization] = @organization
+          hash[:environment] = @environment
+          hash[:sources] = @sources
+          hash[:secrets] = @secrets
+          hash[:stages] = @stages.each_with_object({}) do |(name, stage), h|
+            h[name.to_sym] = Oj.dump(stage)
+          end
+          hash[:sequence] = @sequence
+
+          return hash
+        end
+
+        private
+
+        # Too Many Secrets
+        def setec_astronomy(org, secrets)
+          # Create the RSA private key
+          private_key = OpenSSL::PKey::RSA.new(org.rsa_private_key)
+
+          secrets.hmap do |key, secret|
+            { key => private_key.private_decrypt(Base64.decode64(secret)) }
+          end
+        end
+
+        # Create the ad-hoc StageViews & combine them with the Stages defined
+        # in the Job, returning an array of StageViews & a list of the Stages
+        # in the order to be run.
+        def build_stage_collection(job, org)
+          # Create a JobStage for each ad-hoc stage defined in the job and
+          # add it to the list of stages for this job
+          stages = {}
+          sequence = []
+          begin
+            job[:stages].each do |stage|
+              stage_view = StageView.new(stage)
+              stages[stage_view.name.to_sym] = stage_view
+            end if job.key? :stages
+          rescue StandardError => ex
+            # XXX Probably something wrong with the definition; re-raise it? Or
+            # maybe we get rid of this block and catch it further up (in the
+            # controller?)
+            Cyclid.logger.info "ad-hoc stage creation failed: #{ex}"
+            raise
+          end
+
+          # For each stage in the job, it's either already in the list of
+          # stages because we created on as an ad-hoc stage, or we need to load
+          # it from the database, create a JobStage from it, and add it to the
+          # list of stages
+          job_sequence = job[:sequence]
+          job_sequence.each do |job_stage|
+            job_stage.symbolize_keys!
+
+            raise ArgumentError, 'invalid stage definition' \
+              unless job_stage.key? :stage
+
+            # Store the job in the sequence so that we can run the stages in
+            # the correct order
+            name = job_stage[:stage]
+            sequence << name
+
+            # Try to find the stage
+            if stages.key? name.to_sym
+              # Ad-hoc stage defined in the job
+              stage_view = stages[name.to_sym]
+            else
+              # Try to find a matching pre-defined stage
+              stage = if job_stage.key? :version
+                        org.stages.find_by(name: name, version: job_stage[:version])
+                      else
+                        # If no version given, get the latest
+                        org.stages.where(name: name).last
+                      end
+
+              raise ArgumentError, "stage #{name}:#{version} not found" \
+                if stage.nil?
+
+              stage_view = StageView.new(stage)
+            end
+
+            # Merge in the options specified in this job stage. If the
+            # on_success or on_failure stages are not already in the sequence,
+            # append them to the end.
+            stage_success = { stage: job_stage[:on_success] }
+            job_sequence << stage_success \
+              unless job_stage[:on_success].nil? or \
+                     stage?(job_sequence, job_stage[:on_success])
+            stage_view.on_success = job_stage[:on_success]
+
+            stage_failure = { stage: job_stage[:on_failure] }
+            job_sequence << stage_failure \
+              unless job_stage[:on_failure].nil? or \
+                     stage?(job_sequence, job_stage[:on_failure])
+            stage_view.on_failure = job_stage[:on_failure]
+
+            # Store the modified StageView
+            stages[stage_view.name.to_sym] = stage_view
+          end
+
+          return [stages, sequence]
+        end
+
+        # Search for a stage in the sequence, by name
+        def stage?(sequence, name)
+          found = false
+          sequence.each do |stage|
+            found = stage[:stage] == name || stage['stage'] == name
+            break if found
+          end
+          return found
+        end
+      end
+    end
+  end
+end

data/app/cyclid/job/runner.rb

@@ -0,0 +1,275 @@
+# frozen_string_literal: true
+# Copyright 2016 Liqwyd Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Top level module for the core Cyclid code.
+module Cyclid
+  # Module for the Cyclid API
+  module API
+    # Module for Cyclid Job related classes
+    module Job
+      # Run a job
+      class Runner
+        include Constants::JobStatus
+
+        def initialize(job_id, job_definition, notifier)
+          # The notifier for updating the job status & writing to the log
+          # buffer
+          @notifier = notifier
+
+          # Un-serialize the job
+          begin
+            @job = Oj.load(job_definition, symbol_keys: true)
+
+            environment = @job[:environment]
+            secrets = @job[:secrets]
+          rescue StandardError => ex
+            Cyclid.logger.error "couldn't un-serialize job for job ID #{job_id}: #{ex}"
+            raise 'job failed'
+          end
+
+          # Create an initial job context (more will be added as the job runs)
+          @ctx = @job[:context]
+
+          @ctx[:job_id] = job_id
+          @ctx[:job_name] = @job[:name]
+          @ctx[:job_version] = @job[:version]
+          @ctx[:organization] = @job[:organization]
+          @ctx.merge! environment
+          @ctx.merge! secrets
+
+          begin
+            # We're off!
+            @notifier.status = WAITING
+
+            # Create a Builder
+            @builder = create_builder
+
+            # Obtain a host to run the job on
+            @notifier.write "#{Time.now} : Obtaining build host...\n"
+            @build_host = request_build_host(@builder, environment)
+
+            # We have a build host
+            @notifier.status = STARTED
+
+            # Add some build host details to the build context
+            @ctx.merge! @build_host.context_info
+
+            # Connect a transport to the build host; the notifier is a proxy
+            # to the log buffer
+            @transport = create_transport(@build_host, @notifier)
+
+            # Prepare the host
+            provisioner = create_provisioner(@build_host)
+
+            @notifier.write "#{Time.now} : Preparing build host...\n#{'=' * 79}\n"
+            provisioner.prepare(@transport, @build_host, environment)
+
+            # Check out sources
+            if @job[:sources].any?
+              @notifier.write "#{'=' * 79}\n#{Time.now} : Checking out source...\n"
+              checkout_sources(@transport, @ctx, @job[:sources])
+            end
+          rescue StandardError => ex
+            Cyclid.logger.error "job runner failed: #{ex}"
+
+            @notifier.status = FAILED
+            @notifier.ended = Time.now.to_s
+
+            begin
+              @builder.release(@transport, @build_host) if @build_host
+              @transport.close if @transport
+            rescue ::Net::SSH::Disconnect # rubocop:disable Lint/HandleExceptions
+              # Ignored
+            end
+
+            raise # XXX Raise an internal exception
+          end
+        end
+
+        # Run the stages.
+        #
+        # Start with the first stage, and execute all of the steps until
+        # either one fails, or there are no more steps. The follow the
+        # on_success & on_failure handlers to the next stage. If no
+        # handler is defined, stop.
+        def run
+          status = STARTED
+
+          @notifier.write "#{'=' * 79}\n#{Time.now} : Job started. " \
+                          "Context: #{@ctx.stringify_keys}\n"
+
+          # Run the Job stage actions
+          stages = @job[:stages] || []
+          sequence = (@job[:sequence] || []).first
+
+          # Run each stage in the sequence until there are none left
+          until sequence.nil?
+            # Find the stage
+            raise 'stage not found' unless stages.key? sequence.to_sym
+
+            # Un-serialize the stage into a StageView
+            stage_definition = stages[sequence.to_sym]
+            stage = Oj.load(stage_definition, symbol_keys: true)
+
+            @notifier.write "#{'-' * 79}\n#{Time.now} : " \
+                            "Running stage #{stage.name} v#{stage.version}\n"
+
+            # Run the stage
+            success, rc = run_stage(stage)
+
+            Cyclid.logger.info "stage #{(success ? 'succeeded' : 'failed')} and returned #{rc}"
+
+            # Decide which stage to run next depending on the outcome of this
+            # one
+            if success
+              sequence = stage.on_success
+            else
+              sequence = stage.on_failure
+
+              # Remember the failure while the failure handlers run
+              status = FAILING
+              @notifier.status = status
+            end
+          end
+
+          # Either all of the stages succeeded, and thus the job suceeded, or
+          # (at least one of) the stages failed, and thus the job failed
+          if status == FAILING
+            @notifier.status = FAILED
+            @notifier.ended = Time.now
+            success = false
+          else
+            @notifier.status = SUCCEEDED
+            @notifier.ended = Time.now
+            success = true
+          end
+
+          # We no longer require the build host & transport
+          begin
+            @builder.release(@transport, @build_host)
+            @transport.close
+          rescue ::Net::SSH::Disconnect # rubocop:disable Lint/HandleExceptions
+            # Ignored
+          end
+
+          return success
+        end
+
+        private
+
+        # Create a suitable Builder
+        def create_builder
+          # Each worker creates a new instance
+          builder = Cyclid.builder.new
+          raise "couldn't create a builder" \
+            unless builder
+
+          return builder
+        end
+
+        # Acquire a build host from the builder
+        def request_build_host(builder, environment)
+          # Request a BuildHost
+          build_host = builder.get(environment)
+          raise "couldn't obtain a build host" unless build_host
+
+          return build_host
+        end
+
+        # Find a transport that can be used with the build host, create one and
+        # connect them together
+        def create_transport(build_host, log_buffer)
+          # Create a Transport & connect it to the build host
+          host, username, password, key = build_host.connect_info
+          Cyclid.logger.debug "create_transport: host: #{host} " \
+                                            "username: #{username} " \
+                                            "password: #{password} " \
+                                            "key: #{key}"
+
+          # Try to match a transport that the host supports, to a transport we know how
+          # to create; transports should be listed in the order they're preferred.
+          transport_plugin = nil
+          build_host.transports.each do |t|
+            transport_plugin = Cyclid.plugins.find(t, Cyclid::API::Plugins::Transport)
+          end
+
+          raise "couldn't find a valid transport from #{build_host.transports}" \
+            unless transport_plugin
+
+          # Connect the transport to the build host
+          transport = transport_plugin.new(host: host,
+                                           user: username,
+                                           password: password,
+                                           key: key,
+                                           log: log_buffer)
+          raise 'failed to connect the transport' unless transport
+
+          return transport
+        end
+
+        # Find a provisioner that can be used with the build host and create
+        # one
+        def create_provisioner(build_host)
+          distro = build_host[:distro]
+
+          provisioner_plugin = Cyclid.plugins.find(distro, Cyclid::API::Plugins::Provisioner)
+          raise "couldn't find a valid provisioner for #{distro}" \
+            unless provisioner_plugin
+
+          provisioner = provisioner_plugin.new
+          raise 'failed to create provisioner' unless provisioner
+
+          return provisioner
+        end
+
+        # Find and create a suitable source plugin instance for each source and have it check out
+        # the given source using the transport.
+        def checkout_sources(transport, ctx, sources)
+          sources.each do |job_source|
+            raise 'no type given in source definition' unless job_source.key? :type
+
+            source = Cyclid.plugins.find(job_source[:type], Cyclid::API::Plugins::Source)
+            raise "can't find a plugin for #{job_source[:type]} source" if source.nil?
+
+            success = source.new.checkout(transport, ctx, job_source)
+            raise 'failed to check out source' unless success
+          end
+        end
+
+        # Perform each action defined in the steps of the given stage, until
+        # either an action fails or we run out of steps
+        def run_stage(stage)
+          stage.steps.each do |step|
+            begin
+              # Un-serialize the Action for this step
+              action = Oj.load(step[:action], symbol_keys: true)
+            rescue StandardError
+              Cyclid.logger.error "couldn't un-serialize action for job ID #{job_id}"
+              raise 'job failed'
+            end
+
+            # Run the action
+            action.prepare(transport: @transport, ctx: @ctx)
+            success, rc = action.perform(@notifier)
+
+            return [false, rc] unless success
+          end
+
+          return [true, 0]
+        end
+      end
+    end
+  end
+end