cyclid 0.2.0

data/app/cyclid/plugins/builder/mist.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+# Copyright 2016 Liqwyd Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'mist/config'
+require 'mist/pool'
+require 'mist/client'
+
+# Top level module for the core Cyclid code.
+module Cyclid
+  # Module for the Cyclid API
+  module API
+    # Module for Cyclid Plugins
+    module Plugins
+      # Mist build host
+      class MistHost < BuildHost
+        # SSH is the only acceptable Transport
+        def transports
+          ['ssh']
+        end
+      end
+
+      # Mist builder. Calls out to Mist to obtain a build host instance.
+      class Mist < Builder
+        def initialize
+          mist_config_file = ENV.fetch('MIST_CONFIG', File.join(%w(/ etc mist config)))
+          @config = ::Mist::Config.new(mist_config_file)
+
+          pool = ::Mist::Pool.get(@config.servers)
+          @client = ::Mist::Client.new(pool)
+        end
+
+        # Create & return a build host
+        def get(args = {})
+          args.symbolize_keys!
+
+          Cyclid.logger.debug "mist: args=#{args}"
+
+          # If there is one, split the 'os' into a 'distro' and 'release'
+          if args.key? :os
+            match = args[:os].match(/\A(\w*)_(.*)\Z/)
+            distro = match[1] if match
+            release = match[2] if match
+          else
+            # No OS was specified; use the default
+            # XXX Defaults should be configurable
+            distro = 'ubuntu'
+            release = 'trusty'
+          end
+
+          begin
+            result = @client.call(:create, distro: distro, release: release)
+            Cyclid.logger.debug "mist result=#{result}"
+
+            raise "failed to create build host: #{result['message']}" \
+              unless result['status']
+
+            buildhost = MistHost.new(name: result['name'],
+                                     host: result['ip'],
+                                     username: result['username'],
+                                     workspace: "/home/#{result['username']}",
+                                     password: nil,
+                                     key: @config.ssh_private_key,
+                                     server: result['server'],
+                                     distro: distro,
+                                     release: release)
+          rescue MessagePack::RPC::TimeoutError => ex
+            Cyclid.logger.error "Mist create call timedout: #{ex}"
+            raise "mist failed: #{ex}"
+          rescue StandardError => ex
+            Cyclid.logger.error "couldn't get a build host from Mist: #{ex}"
+            raise "mist failed: #{ex}"
+          end
+
+          Cyclid.logger.debug "mist buildhost=#{buildhost.inspect}"
+          return buildhost
+        end
+
+        # Destroy the build host
+        def release(_transport, buildhost)
+          name = buildhost[:name]
+          server = buildhost[:server]
+
+          begin
+            @client.call(:destroy, name: name, server: server)
+          rescue MessagePack::RPC::TimeoutError => ex
+            Cyclid.logger.error "Mist destroy timed out: #{ex}"
+          end
+        end
+
+        # Register this plugin
+        register_plugin 'mist'
+      end
+    end
+  end
+end

data/app/cyclid/plugins/dispatcher.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+# Copyright 2016 Liqwyd Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Top level module for the core Cyclid code.
+module Cyclid
+  # Module for the Cyclid API
+  module API
+    # Module for Cyclid Plugins
+    module Plugins
+      # Base class for Dispatcher
+      class Dispatcher < Base
+        # Return the 'human' name for the plugin type
+        def self.human_name
+          'dispatcher'
+        end
+
+        # Dispatch a job to a worker and return some opaque data that can be
+        # used to identify that job (E.g. an ID, UUID etc.)
+        def dispatch(job, record, callback = nil)
+        end
+      end
+
+      # A Runner may be running locally (within the API application context)
+      # or remotely. A job runner needs to send updates about the job status
+      # but obviously, a remote runner can't just update the JobRecord
+      # directly: they may put a message on a queue, which a job at the API
+      # application would consume and update the JobRecord.
+      #
+      # A Notifier provides an abstract method to update the JobRecord
+      # status and can also proxy LogBuffer writes.
+      #
+      module Notifier
+        # Base class for Notifiers
+        class Base
+          def initialize(job_id, callback_object)
+          end
+
+          # Update the JobRecord status
+          def status=(status)
+          end
+
+          # Update the JobRecord ended field
+          def ended=(time)
+          end
+
+          # Notify any callbacks that the job has completed
+          def completion(success)
+          end
+
+          # Proxy data to the log buffer
+          def write(data)
+          end
+        end
+
+        # Plugins may create a Callback instance that contains callbacks which
+        # are called by the Notifier when something happens; the Plugin can
+        # then take whatever action they need (E.g. updating an external
+        # status)
+        class Callback
+          # Called when the job completes
+          def completion(_job_id, _status)
+          end
+
+          # Called whenever the job status changes
+          def status_changed(_job_id, _status)
+          end
+
+          # Called whenever any data is written to the job record log
+          def log_write(_job_id, _data)
+          end
+        end
+      end
+    end
+  end
+end
+
+require_rel 'dispatcher/*.rb'

data/app/cyclid/plugins/dispatcher/local.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+# Copyright 2016 Liqwyd Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'sidekiq'
+
+# Top level module for the core Cyclid code.
+module Cyclid
+  # Module for the Cyclid API
+  module API
+    # Module for Cyclid Plugins
+    module Plugins
+      # Local Sidekiq based dispatcher
+      class Local < Dispatcher
+        # Queue the job to be run asynchronously.
+        def dispatch(job, record, callback = nil)
+          Cyclid.logger.debug "dispatching job: #{job}"
+
+          job_definition = job.to_hash.to_json
+
+          record.job_name = job.name
+          record.job_version = job.version
+          record.job = job_definition
+          record.save!
+
+          # The callback instance has to be serailised into JSON to survive the
+          # trip through Redis to Sidekiq
+          callback_json = callback.nil? ? nil : Oj.dump(callback)
+
+          # Create a SideKiq worker and pass in the job
+          Worker::Local.perform_async(job_definition, record.id, callback_json)
+
+          # The JobRecord ID is as good a job identifier as anything
+          return record.id
+        end
+
+        # Healthcheck; ensure that Sinatra is available and not under duress
+        require 'sidekiq/api'
+        extend Health::Helpers
+
+        # Perform a health check; for this plugin that means:
+        #
+        # Is Sidekiq running?
+        # Is the queue size healthy?
+        def self.status
+          stats = Sidekiq::Stats.new
+          if stats.processes_size.zero?
+            health_status(:error,
+                          'no Sidekiq process is running')
+          elsif stats.enqueued > 10
+            health_status(:warning,
+                          "Sidekiq queue length is too high: #{stats.enqueued}")
+          elsif stats.default_queue_latency > 60
+            health_status(:warning,
+                          "Sidekiq queue latency is too high: #{stats.default_queue_latency}")
+          else
+            health_status(:ok,
+                          'sidekiq is okay')
+          end
+        end
+
+        # Register this plugin
+        register_plugin 'local'
+      end
+
+      # A Runner may be running locally (within the API application context)
+      # or remotely. A job runner needs to send updates about the job status
+      # but obviously, a remote runner can't just update the JobRecord
+      # directly: they may put a message on a queue, which a job at the API
+      # application would consume and update the JobRecord.
+      #
+      # A Notifier provides an abstract method to update the JobRecord
+      # status and can also proxy LogBuffer writes.
+      #
+      module Notifier
+        # This is a local Notifier, so it can just pass updates directly on to
+        # the JobRecord & LogBuffer
+        class Local < Base
+          def initialize(job_id, callback)
+            @job_id = job_id
+            @job_record = JobRecord.find(job_id)
+
+            # Create a LogBuffer
+            @log_buffer = LogBuffer.new(@job_record)
+
+            @callback = callback
+          end
+
+          # Set the JobRecord status
+          def status=(status)
+            @job_record.status = status
+            @job_record.save!
+
+            # Ping the callback status_changed hook, if required
+            @callback.status_changed(@job_id, status) if @callback
+          end
+
+          # Set the JobRecord ended
+          def ended=(time)
+            @job_record.ended = time
+            @job_record.save!
+          end
+
+          # Ping the callback completion hook, if required
+          def completion(success)
+            @callback.completion(@job_id, success) if @callback
+          end
+
+          # Write data to the log buffer
+          def write(data)
+            @log_buffer.write data
+
+            # Ping the callback log_write hook, if required
+            @callback.log_write(@job_id, data) if @callback
+          end
+        end
+      end
+
+      # Namespace for any asyncronous workers
+      module Worker
+        # Local Sidekiq based worker
+        class Local
+          include Sidekiq::Worker
+
+          sidekiq_options retry: false
+
+          # Run a job Runner asynchronously
+          def perform(job, job_id, callback_object)
+            begin
+              # Unserialize the callback object, if there is one
+              callback = callback_object.nil? ? nil : Oj.load(callback_object)
+
+              notifier = Notifier::Local.new(job_id, callback)
+            rescue StandardError => ex
+              Cyclid.logger.debug "couldn't create notifier: #{ex}"
+              return false
+            end
+
+            begin
+              runner = Cyclid::API::Job::Runner.new(job_id, job, notifier)
+              success = runner.run
+            rescue StandardError => ex
+              Cyclid.logger.error "job runner failed: #{ex}"
+              success = false
+            end
+
+            # Notify completion
+            notifier.completion(success)
+
+            return success
+          end
+        end
+      end
+    end
+  end
+end

data/app/cyclid/plugins/provisioner.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+# Copyright 2016 Liqwyd Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Top level module for the core Cyclid code.
+module Cyclid
+  # Module for the Cyclid API
+  module API
+    # Module for Cyclid Plugins
+    module Plugins
+      # Base class for Provisioner plugins
+      class Provisioner < Base
+        # Return the 'human' name for the plugin type
+        def self.human_name
+          'provisioner'
+        end
+
+        # Process the environment, performing all of the steps necasary to
+        # configure the host according to the given environment; this can
+        # include adding repositories, installing packages etc.
+        def prepare(_transport, _buildhost, _env = {})
+          false
+        end
+      end
+    end
+  end
+end
+
+require_rel 'provisioner/*.rb'

data/app/cyclid/plugins/provisioner/debian.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+# Copyright 2016 Liqwyd Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Top level module for the core Cyclid code.
+module Cyclid
+  # Module for the Cyclid API
+  module API
+    # Module for Cyclid Plugins
+    module Plugins
+      # Debian provisioner
+      class Debian < Provisioner
+        # Prepare a Debian based build host
+        def prepare(transport, buildhost, env = {})
+          transport.export_env('DEBIAN_FRONTEND' => 'noninteractive')
+
+          if env.key? :repos
+            env[:repos].each do |repo|
+              next unless repo.key? :url
+
+              url = repo[:url]
+              match = url.match(/\A(http|https):.*\Z/)
+              next unless match
+
+              case match[1]
+              when 'http', 'https'
+                add_http_repository(transport, url, repo, buildhost)
+              end
+            end
+
+            success = transport.exec 'sudo apt-get update'
+            raise 'failed to update repositories' unless success
+          end
+
+          env[:packages].each do |package|
+            success = transport.exec \
+              "sudo -E apt-get install -y #{package}"
+            raise "failed to install package #{package}" unless success
+          end if env.key? :packages
+        rescue StandardError => ex
+          Cyclid.logger.error "failed to provision #{buildhost[:name]}: #{ex}"
+          raise
+        end
+
+        private
+
+        def add_http_repository(transport, url, repo, buildhost)
+          raise 'an HTTP repository must provide a list of components' \
+            unless repo.key? :components
+
+          # Create a sources.list.d fragment
+          release = buildhost[:release]
+          components = repo[:components]
+          fragment = "deb #{url} #{release} #{components}"
+
+          success = transport.exec \
+            "echo '#{fragment}' | sudo tee -a /etc/apt/sources.list.d/cyclid.list"
+          raise "failed to add repository #{url}" unless success
+
+          if repo.key? :key_id
+            # Import the signing key
+            key_id = repo[:key_id]
+
+            success = transport.exec \
+              "gpg --keyserver keyserver.ubuntu.com --recv-keys #{key_id}"
+            raise "failed to import key #{key_id}" unless success
+
+            success = transport.exec \
+              "gpg -a --export #{key_id} | sudo apt-key add -"
+            raise "failed to add repository key #{key_id}" unless success
+          end
+        end
+
+        # Register this plugin
+        register_plugin 'debian'
+      end
+    end
+  end
+end