cyclid 0.2.0

data/app/cyclid/plugins/provisioner/ubuntu.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+# Copyright 2016 Liqwyd Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Top level module for the core Cyclid code.
+module Cyclid
+  # Module for the Cyclid API
+  module API
+    # Module for Cyclid Plugins
+    module Plugins
+      # Ubuntu provisioner
+      class Ubuntu < Provisioner
+        # Prepare an Ubuntu based build host
+        def prepare(transport, buildhost, env = {})
+          transport.export_env('DEBIAN_FRONTEND' => 'noninteractive')
+
+          if env.key? :repos
+            env[:repos].each do |repo|
+              next unless repo.key? :url
+
+              # Check if it's a PPA, or a traditional repository
+              url = repo[:url]
+              match = url.match(/\A(ppa|http|https):.*\Z/)
+              next unless match
+
+              case match[1]
+              when 'ppa'
+                add_ppa_repository(transport, url)
+              when 'http', 'https'
+                add_http_repository(transport, url, repo, buildhost)
+              end
+            end
+
+            success = transport.exec 'sudo apt-get update'
+            raise 'failed to update repositories' unless success
+          end
+
+          env[:packages].each do |package|
+            success = transport.exec \
+              "sudo -E apt-get install -y #{package}"
+            raise "failed to install package #{package}" unless success
+          end if env.key? :packages
+        rescue StandardError => ex
+          Cyclid.logger.error "failed to provision #{buildhost[:name]}: #{ex}"
+          raise
+        end
+
+        private
+
+        def add_ppa_repository(transport, url)
+          success = transport.exec "sudo apt-add-repository -y #{url}"
+          raise "failed to add repository #{url}" unless success
+        end
+
+        def add_http_repository(transport, url, repo, buildhost)
+          raise 'an HTTP repository must provide a list of components' \
+            unless repo.key? :components
+
+          # Create a sources.list.d fragment
+          release = buildhost[:release]
+          components = repo[:components]
+          fragment = "deb #{url} #{release} #{components}"
+
+          success = transport.exec \
+            "echo '#{fragment}' | sudo tee -a /etc/apt/sources.list.d/cyclid.list"
+          raise "failed to add repository #{url}" unless success
+
+          if repo.key? :key_id
+            # Import the signing key
+            key_id = repo[:key_id]
+
+            success = transport.exec \
+              "gpg --keyserver keyserver.ubuntu.com --recv-keys #{key_id}"
+            raise "failed to import key #{key_id}" unless success
+
+            success = transport.exec \
+              "gpg -a --export #{key_id} | sudo apt-key add -"
+            raise "failed to add repository key #{key_id}" unless success
+          end
+        end
+
+        # Register this plugin
+        register_plugin 'ubuntu'
+      end
+    end
+  end
+end

data/app/cyclid/plugins/source.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+# Copyright 2016 Liqwyd Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Top level module for the core Cyclid code.
+module Cyclid
+  # Module for the Cyclid API
+  module API
+    # Module for Cyclid Plugins
+    module Plugins
+      # Base class for Source plugins
+      class Source < Base
+        # Return the 'human' name for the plugin type
+        def self.human_name
+          'source'
+        end
+
+        # Process the source to produce a copy of the remote code in a
+        # directory in the working directory
+        def checkout(_transport, _ctx, _source = {})
+          false
+        end
+      end
+    end
+  end
+end
+
+require_rel 'source/*.rb'

data/app/cyclid/plugins/source/git.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+# Copyright 2016 Liqwyd Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Top level module for the core Cyclid code.
+module Cyclid
+  # Module for the Cyclid API
+  module API
+    # Module for Cyclid Plugins
+    module Plugins
+      # Git source plugin
+      class Git < Source
+        # Run commands via. the transport to check out a given Git remote
+        # repository
+        def checkout(transport, ctx, source = {})
+          source.symbolize_keys!
+
+          raise 'invalid git source definition' \
+            unless source.key? :url
+
+          # Add any context data (which could include secrets)
+          source = source.interpolate(ctx)
+
+          url = URI(source[:url])
+
+          # If the source includes an OAuth token, add it to the URL as the
+          # username
+          url.user = source[:token] if source.key? :token
+
+          success = transport.exec "git clone #{url}"
+          return false unless success
+
+          # If a branch was given, check it out
+          if source.key? :branch
+            branch = source[:branch]
+
+            match = url.path.match(%r{^.*\/(\w*)})
+            source_dir = "#{ctx[:workspace]}/#{match[1]}"
+
+            success = transport.exec("git fetch origin #{branch}:#{branch}", source_dir)
+            success = transport.exec("git checkout #{branch}", source_dir) \
+              unless success == false
+          end
+
+          return success
+        end
+
+        # Register this plugin
+        register_plugin 'git'
+      end
+    end
+  end
+end

data/app/cyclid/plugins/transport.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+# Copyright 2016 Liqwyd Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Top level module for the core Cyclid code.
+module Cyclid
+  # Module for the Cyclid API
+  module API
+    # Module for Cyclid Plugins
+    module Plugins
+      # Base class for Transport plugins
+      class Transport < Base
+        def initialize(args = {})
+        end
+
+        # Return the 'human' name for the plugin type
+        def self.human_name
+          'transport'
+        end
+
+        # If possible, export each of the variables in env as a shell
+        # environment variables. The default is simply to remember the
+        # environment variables, which will be exported each time when a
+        # command is run.
+        def export_env(env = {})
+          @env = env
+        end
+
+        # Run a command on the remote host.
+        def exec(_cmd, _path = nil)
+          false
+        end
+
+        # Copy data from a local IO object to a remote file.
+        def upload(_io, _path)
+          false
+        end
+
+        # Copy a data from remote file to a local IO object
+        def download(_io, _path)
+          false
+        end
+
+        # Disconnect the transport
+        def close
+        end
+      end
+    end
+  end
+end
+
+require_rel 'transport/*.rb'

data/app/cyclid/plugins/transport/ssh.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+# Copyright 2016 Liqwyd Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'net/ssh'
+require 'net/scp'
+require 'cyclid/log_buffer'
+
+# Top level module for the core Cyclid code.
+module Cyclid
+  # Module for the Cyclid API
+  module API
+    # Module for Cyclid Plugins
+    module Plugins
+      # SSH based transport
+      class Ssh < Transport
+        attr_reader :exit_code, :exit_signal
+
+        def initialize(args = {})
+          args.symbolize_keys!
+
+          # Hostname, username & a log target are required
+          return false unless args.include? :host and \
+                              args.include? :user and \
+                              args.include? :log
+
+          password = args[:password] if args.include? :password
+          keys = [args[:key]] if args.include? :key
+
+          @log = args[:log]
+
+          # Create an SSH channel
+          Cyclid.logger.debug 'waiting for SSH...'
+
+          start = Time.now
+          loop do
+            begin
+              @session = Net::SSH.start(args[:host],
+                                        args[:user],
+                                        password: password,
+                                        keys: keys,
+                                        timeout: 5)
+              break unless @session.nil?
+            rescue Net::SSH::AuthenticationFailed
+              Cyclid.logger.debug 'SSH authentication failed'
+            end
+
+            sleep 5
+
+            raise 'timed out waiting for SSH' \
+              if (Time.now - start) >= 30
+          end
+        end
+
+        # Execute a command via. SSH
+        def exec(cmd, path = nil)
+          command = build_command(cmd, path, @env)
+          Cyclid.logger.debug "command=#{command}"
+
+          @session.open_channel do |channel|
+            channel.on_open_failed do |_ch, _code, desc|
+              # XXX raise
+              abort "failed to open channel: #{desc}"
+            end
+
+            # STDOUT
+            channel.on_data do |_ch, data|
+              # Send to Log Buffer
+              @log.write data
+            end
+
+            # STDERR
+            channel.on_extended_data do |_ch, _type, data|
+              # Send to Log Buffer
+              @log.write data
+            end
+
+            # Capture return value from commands
+            channel.on_request 'exit-status' do |_ch, data|
+              @exit_code = data.read_long
+            end
+
+            # Capture a command exiting with a signal
+            channel.on_request 'exit-signal' do |_ch, data|
+              @exit_signal = data.read_long
+            end
+
+            channel.exec command do |_ch, _success|
+            end
+          end
+
+          # Run the SSH even loop; this blocks until the command has completed
+          @session.loop
+
+          @exit_code.zero? && @exit_signal.nil? ? true : false
+        end
+
+        # Copy data from a local IO object to a remote file via. SCP
+        def upload(io, path)
+          channel = @session.scp.upload io, path
+          channel.wait
+        end
+
+        # Copy a data from remote file to a local IO object
+        def download(io, path)
+          channel = @session.scp.download path, io
+          channel.wait
+        end
+
+        # Close the SSH connection
+        def close
+          logout
+
+          @session.close
+        end
+
+        private
+
+        def logout
+          exec 'exit'
+        end
+
+        def build_command(cmd, path = nil, env = {})
+          command = []
+          if env
+            vars = env.map do |k, value|
+              key = k.upcase
+              key.gsub!(/\s/, '_')
+              "export #{key}=\"#{value}\""
+            end
+            command << vars.join(';')
+          end
+
+          command << "cd #{path}" if path
+          command << cmd
+          command.join(';')
+        end
+
+        # Register this plugin
+        register_plugin 'ssh'
+      end
+    end
+  end
+end

data/app/cyclid/sinatra/api_helpers.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+# Copyright 2016 Liqwyd Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'oj'
+require 'yaml'
+
+# Top level module for the core Cyclid code.
+module Cyclid
+  # Module for the Cyclid API
+  module API
+    # Sinatra helpers
+    module APIHelpers
+      # Safely parse & validate the request body
+      def parse_request_body
+        # Parse the the request
+        begin
+          request.body.rewind
+
+          if request.content_type == 'application/json' or \
+             request.content_type == 'text/json'
+
+            data = Oj.load request.body.read
+          elsif request.content_type == 'application/x-yaml' or \
+                request.content_type == 'text/x-yaml'
+
+            data = YAML.load request.body.read
+          else
+            halt_with_json_response(415, \
+                                    Errors::HTTPErrors::INVALID_JSON, \
+                                    "unsupported content type #{request.content_type}")
+          end
+        rescue Oj::ParseError, YAML::Exception => ex
+          Cyclid.logger.debug ex.message
+          halt_with_json_response(400, Errors::HTTPErrors::INVALID_JSON, ex.message)
+        end
+
+        # Sanity check the request
+        halt_with_json_response(400, \
+                                Errors::HTTPErrors::INVALID_JSON, \
+                                'request body can not be empty') if data.nil?
+        halt_with_json_response(400, \
+                                Errors::HTTPErrors::INVALID_JSON, \
+                                'request body is invalid') unless data.is_a?(Hash)
+
+        return data
+      end
+
+      # Return a RESTful JSON response
+      def json_response(id, description)
+        Oj.dump('id' => id, 'description' => description)
+      end
+    end
+  end
+end