capsulecd 1.0.0

data/lib/capsulecd/base/source/github.rb

@@ -0,0 +1,183 @@
+require 'octokit'
+require 'uri'
+require 'git'
+require 'capsulecd'
+require 'pp'
+
+module CapsuleCD
+  module Source
+    module Github
+      # all of these instance variables are available for use within hooks
+      attr_accessor :source_client
+      attr_accessor :source_git_base_info
+      attr_accessor :source_git_head_info
+      attr_accessor :source_git_parent_path
+      attr_accessor :source_git_local_path
+      attr_accessor :source_git_local_branch
+      attr_accessor :source_git_remote
+      attr_accessor :source_release_commit
+      attr_accessor :source_release_artifacts
+
+      # define the Source API methods
+
+      # configure method will generate an authenticated client that can be used to comunicate with Github
+      # MUST set @source_git_parent_path
+      # MUST set @source_client
+      def source_configure
+        puts 'github source_configure'
+        fail CapsuleCD::Error::SourceAuthenticationFailed, 'Missing github access token' unless @config.source_github_access_token
+
+        @source_release_commit = nil
+        @source_release_artifacts = []
+
+        @source_git_parent_path = @config.source_git_parent_path || Dir.mktmpdir
+        Octokit.auto_paginate = true
+        Octokit.configure do |c|
+          c.api_endpoint = @config.source_github_api_endpoint if @config.source_github_api_endpoint
+          c.web_endpoint = @config.source_github_web_endpoint if @config.source_github_web_endpoint
+        end
+        @source_client = Octokit::Client.new(access_token: @config.source_github_access_token)
+      end
+
+      # all capsule CD processing will be kicked off via a payload. In Github's case, the payload is the webhook data.
+      # should check if the pull request opener even has permissions to create a release.
+      # all sources should process the payload by downloading a git repository that contains the master branch merged with the test branch
+      # MUST set source_git_local_path
+      # MUST set source_git_local_branch
+      # MUST set source_git_head_info
+      # REQUIRES source_git_parent_path
+      def source_process_push_payload(payload)
+        # set the processed head info
+        @source_git_head_info = payload['head']
+        CapsuleCD::ValidationUtils.validate_repo_payload(@source_git_head_info)
+
+        # set the remote url, with embedded token
+        uri = URI.parse(@source_git_head_info['repo']['clone_url'])
+        uri.user = @config.source_github_access_token
+        @source_git_remote = uri.to_s
+        @source_git_local_branch = @source_git_head_info['repo']['ref']
+        # clone the merged branch
+        # https://sethvargo.com/checkout-a-github-pull-request/
+        # https://coderwall.com/p/z5rkga/github-checkout-a-pull-request-as-a-branch
+        @source_git_local_path = CapsuleCD::GitUtils.clone(@source_git_parent_path, @source_git_head_info['repo']['name'], @source_git_remote)
+        CapsuleCD::GitUtils.checkout(@source_git_local_path, @source_git_head_info['repo']['sha1'])
+      end
+
+      # all capsule CD processing will be kicked off via a payload. In Github's case, the payload is the pull request data.
+      # should check if the pull request opener even has permissions to create a release.
+      # all sources should process the payload by downloading a git repository that contains the master branch merged with the test branch
+      # MUST set source_git_local_path
+      # MUST set source_git_local_branch
+      # MUST set source_git_base_info
+      # MUST set source_git_head_info
+      # REQUIRES source_client
+      # REQUIRES source_git_parent_path
+      def source_process_pull_request_payload(payload)
+        puts 'github source_process_payload'
+
+        # validate the github specific payload options
+        unless (payload['state'] == 'open')
+          fail CapsuleCD::Error::SourcePayloadUnsupported, 'Pull request has an invalid action'
+        end
+        unless (payload['base']['repo']['default_branch'] == payload['base']['ref'])
+          fail CapsuleCD::Error::SourcePayloadUnsupported, 'Pull request is not being created against the default branch of this repository (usually master)'
+        end
+        # check the payload push user.
+
+        # TODO: figure out how to do optional authenication. possible options, Source USER, token based auth, no auth when used with capsulecd.com.
+        # unless @source_client.collaborator?(payload['base']['repo']['full_name'], payload['user']['login'])
+        #
+        #   @source_client.add_comment(payload['base']['repo']['full_name'], payload['number'], CapsuleCD::BotUtils.pull_request_comment)
+        #   fail CapsuleCD::Error::SourceUnauthorizedUser, 'Pull request was opened by an unauthorized user'
+        # end
+
+        # set the processed base/head info,
+        @source_git_base_info = payload['base']
+        @source_git_head_info = payload['head']
+        CapsuleCD::ValidationUtils.validate_repo_payload(@source_git_base_info)
+        CapsuleCD::ValidationUtils.validate_repo_payload(@source_git_head_info)
+
+        # set the remote url, with embedded token
+        uri = URI.parse(payload['base']['repo']['clone_url'])
+        uri.user = @config.source_github_access_token
+        @source_git_remote = uri.to_s
+
+        # clone the merged branch
+        # https://sethvargo.com/checkout-a-github-pull-request/
+        # https://coderwall.com/p/z5rkga/github-checkout-a-pull-request-as-a-branch
+        @source_git_local_path = CapsuleCD::GitUtils.clone(@source_git_parent_path, @source_git_head_info['repo']['name'], @source_git_remote)
+        @source_git_local_branch = "pr_#{payload['number']}"
+        CapsuleCD::GitUtils.fetch(@source_git_local_path, "refs/pull/#{payload['number']}/merge", @source_git_local_branch)
+        CapsuleCD::GitUtils.checkout(@source_git_local_path, @source_git_local_branch)
+
+        # show a processing message on the github PR.
+        @source_client.create_status(payload['base']['repo']['full_name'], @source_git_head_info['sha'], 'pending',
+                                     context: 'CapsuleCD',
+                                     target_url: 'http://www.github.com/AnalogJ/capsulecd',
+                                     description: 'Started processing package. Pull request will be merged automatically when complete.')
+      end
+
+      # REQUIRES source_client
+      # REQUIRES source_release_commit
+      # REQUIRES source_git_local_path
+      # REQUIRES source_git_local_branch
+      # REQUIRES source_git_base_info
+      # REQUIRES source_git_head_info
+      # REQUIRES source_release_artifacts
+      # REQUIRES source_git_parent_path
+      def source_release
+        puts 'github source_release'
+
+        # push the version bumped metadata file + newly created files to
+        CapsuleCD::GitUtils.push(@source_git_local_path, @source_git_local_branch, @source_git_base_info['ref'])
+        # sleep because github needs time to process the new tag.
+        sleep 5
+
+        # calculate the release sha
+        release_sha = ('0' * (40 - @source_release_commit.sha.strip.length)) + @source_release_commit.sha.strip
+
+        # get the release changelog
+        release_body = CapsuleCD::GitUtils.generate_changelog(@source_git_local_path, @source_git_base_info['sha'], @source_git_head_info['sha'], @source_git_base_info['repo']['full_name'])
+
+        release = @source_client.create_release(@source_git_base_info['repo']['full_name'], @source_release_commit.name,       target_commitish: release_sha,
+                                                                                                                               name: @source_release_commit.name,
+                                                                                                                               body: release_body)
+
+        @source_release_artifacts.each do |release_artifact|
+          @source_client.upload_asset(release[:url], release_artifact[:path], name: release_artifact[:name])
+        end
+
+        FileUtils.remove_entry_secure @source_git_parent_path if Dir.exists?(@source_git_parent_path)
+        # set the pull request status
+        @source_client.create_status(@source_git_base_info['repo']['full_name'], @source_git_head_info['sha'], 'success',
+                                     context: 'CapsuleCD',
+                                     target_url: 'http://www.github.com/AnalogJ/capsulecd',
+                                     description: 'Pull-request was successfully merged, new release created.')
+      end
+
+      # requires @source_client
+      # requires @source_git_parent_path
+      # requires @source_git_base_info
+      # requires @source_git_head_info
+      def source_notify(step, status='pending')
+
+        @source_client.create_status(@source_git_base_info['repo']['full_name'], @source_git_head_info['sha'], status,
+          context: 'CapsuleCD',
+          target_url: 'http://www.github.com/AnalogJ/capsulecd',
+          description: "Started '#{step}' step. Pull request will be merged automatically when complete.")
+
+        yield
+
+      rescue => ex
+        puts 'github source_process_failure'
+        FileUtils.remove_entry_secure @source_git_parent_path if Dir.exists?(@source_git_parent_path)
+        @source_client.create_status(@source_git_base_info['repo']['full_name'], @source_git_head_info['sha'], 'failure',
+                                     context: 'CapsuleCD',
+                                     target_url: 'http://www.github.com/AnalogJ/capsulecd',
+                                     description: ex.message.slice!(0..135))
+        raise
+      end
+
+    end
+  end
+end

data/lib/capsulecd/base/transform_engine.rb

@@ -0,0 +1,62 @@
+require 'yaml'
+module CapsuleCD
+  module EngineExtension
+  end
+
+  class TransformEngine
+    def initialize()
+    end
+
+    def transform(engine, config_file, type = :repo) #type can only be :repo or :global
+      @engine = engine
+      @type = type
+
+      unless File.exists?(config_file)
+        puts 'no configuration file found, no engine hooks'
+        return
+      end
+
+      #parse the config file and generate a module file that we can use as part of our engine
+      @config = YAML.load(File.open(config_file).read)
+      populate_engine_extension
+      register_extension
+    end
+
+    def populate_engine_extension()
+      # lets loop though all the keys, and raise an error if the hooks specified are not available.
+      if @type == :repo
+        @config.keys.each do |key|
+           fail CapsuleCD::Error::EngineTransformUnavailableStep, key + ' cannot be overridden by repo capsule.yml file.' if %w(source_configure source_process_pull_request_payload source_process_push_payload runner_retrieve_payload).include?(key)
+        end
+      end
+
+      # yeah yeah, metaprogramming is evil. But actually its really just a powerful tool, albeit a really complicated one.
+      # Like any tool, you can use it incorrectly. With great power comes.. yada yada.
+      # In general metaprogramming is bad because it makes it hard to reason about your code. In this case we're using it
+      # to allow other developers to override our engine steps, and/or attach to our hooks. In this case its their
+      # responsibility not to f*&^ it all up.
+
+      # http://www.monkeyandcrow.com/blog/building_classes_dynamically/
+      # https://rubymonk.com/learning/books/5-metaprogramming-ruby-ascent/chapters/24-eval/lessons/68-class-eval
+      # https://www.ruby-forum.com/topic/207350
+      @config.each do |step, value|
+        next unless %w(source_configure source_process_pull_request_payload source_process_push_payload runner_retrieve_payload build_step test_step package_step source_release release_step).include?(step)
+
+        value.each do |prefix, method_script|
+          EngineExtension.class_eval(<<-METHOD
+          def #{prefix == 'override' ? '' : prefix+ '_'}#{step};
+            #{method_script};
+          end
+METHOD
+)
+        end
+      end
+    end
+
+    # at this point the EngineExtension module should be populated with all the hooks and methods.
+    # now we need to add them to the engine.
+    def register_extension
+      @engine.class.prepend(CapsuleCD::EngineExtension)
+    end
+  end
+end

data/lib/capsulecd/chef/chef_engine.rb

@@ -0,0 +1,172 @@
+require 'semverly'
+require 'open3'
+require 'bundler'
+require_relative '../base/engine'
+require_relative 'chef_helper'
+require 'base64'
+require 'fileutils'
+
+module CapsuleCD
+  module Chef
+    class ChefEngine < Engine
+      def build_step
+        super
+        # validate that the chef metadata.rb file exists
+
+        unless File.exist?(@source_git_local_path + '/metadata.rb')
+          fail CapsuleCD::Error::BuildPackageInvalid, 'metadata.rb file is required to process Chef cookbook'
+        end
+
+        # bump up the chef cookbook version
+        metadata_str = CapsuleCD::Chef::ChefHelper.read_repo_metadata(@source_git_local_path)
+        chef_metadata = CapsuleCD::Chef::ChefHelper.parse_metadata(metadata_str)
+        next_version = bump_version(SemVer.parse(chef_metadata.version))
+
+        new_metadata_str = metadata_str.gsub(/(version\s+['"])[0-9\.]+(['"])/, "\\1#{next_version}\\2")
+        CapsuleCD::Chef::ChefHelper.write_repo_metadata(@source_git_local_path, new_metadata_str)
+
+        # TODO: check if this cookbook name and version already exist.
+
+        # check for/create any required missing folders/files
+        # Berksfile.lock and Gemfile.lock are not required to be commited, but they should be.
+        unless File.exist?(@source_git_local_path + '/Rakefile')
+          File.open(@source_git_local_path + '/Rakefile', 'w') { |file| file.write('task :test') }
+        end
+        unless File.exist?(@source_git_local_path + '/Berksfile')
+          File.open(@source_git_local_path + '/Berksfile', 'w') { |file| file.write('site :opscode') }
+        end
+        unless File.exist?(@source_git_local_path + '/Gemfile')
+          File.open(@source_git_local_path + '/Gemfile', 'w') { |file| file.write('source "https://rubygems.org"') }
+        end
+        unless File.exist?(@source_git_local_path + '/spec')
+          FileUtils.mkdir(@source_git_local_path + '/spec')
+        end
+        unless File.exist?(@source_git_local_path + '/.gitignore')
+          CapsuleCD::GitUtils.create_gitignore(@source_git_local_path, ['ChefCookbook'])
+        end
+      end
+
+      def test_step
+        super
+
+        # the cookbook has already been downloaded. lets make sure all its dependencies are available.
+        Open3.popen3('berks install', chdir: @source_git_local_path) do |_stdin, stdout, stderr, external|
+          { stdout: stdout, stderr: stderr }. each do |name, stream_buffer|
+            Thread.new do
+              until (line = stream_buffer.gets).nil?
+                puts "#{name} -> #{line}"
+              end
+            end
+          end
+          # wait for process
+          external.join
+          unless external.value.success?
+            fail CapsuleCD::Error::TestDependenciesError, 'berks install failed. Check cookbook dependencies'
+          end
+        end
+
+        # lets download all its gem dependencies
+        Bundler.with_clean_env do
+          Open3.popen3('bundle install', chdir: @source_git_local_path) do |_stdin, stdout, stderr, external|
+            { stdout: stdout, stderr: stderr }. each do |name, stream_buffer|
+              Thread.new do
+                until (line = stream_buffer.gets).nil?
+                  puts "#{name} -> #{line}"
+                end
+              end
+            end
+            # wait for process
+            external.join
+            unless external.value.success?
+              fail CapsuleCD::Error::TestDependenciesError, 'bundle install failed. Check gem dependencies'
+            end
+          end
+
+          # run test command
+          test_cmd = @config.engine_cmd_test || 'rake test'
+          Open3.popen3(test_cmd, chdir: @source_git_local_path) do |_stdin, stdout, stderr, external|
+            { stdout: stdout, stderr: stderr }. each do |name, stream_buffer|
+              Thread.new do
+                until (line = stream_buffer.gets).nil?
+                  puts "#{name} -> #{line}"
+                end
+              end
+            end
+            # wait for process
+            external.join
+            unless external.value.success?
+              fail CapsuleCD::Error::TestRunnerError, test_cmd + ' failed. Check log for exact error'
+            end
+          end unless @config.engine_disable_test
+        end
+      end
+
+      def package_step
+        super
+        metadata_str = CapsuleCD::Chef::ChefHelper.read_repo_metadata(@source_git_local_path)
+        chef_metadata = CapsuleCD::Chef::ChefHelper.parse_metadata(metadata_str)
+        next_version = SemVer.parse(chef_metadata.version)
+        # commit changes to the cookbook. (test run occurs before this, and it should clean up any instrumentation files, created,
+        # as they will be included in the commmit and any release artifacts)
+        CapsuleCD::GitUtils.commit(@source_git_local_path, "(v#{next_version}) Automated packaging of release by CapsuleCD")
+        @source_release_commit = CapsuleCD::GitUtils.tag(@source_git_local_path, "v#{next_version}")
+      end
+
+      # this step should push the release to the package repository (ie. npm, chef supermarket, rubygems)
+      def release_step
+        super
+        puts @source_git_parent_path
+        pem_path = File.expand_path('~/client.pem')
+        knife_path = File.expand_path('~/knife.rb')
+
+        unless @config.chef_supermarket_username || @config.chef_supermarket_key
+          fail CapsuleCD::Error::ReleaseCredentialsMissing, 'cannot deploy cookbook to supermarket, credentials missing'
+          return
+        end
+
+        # knife is really sensitive to folder names. The cookbook name MUST match the folder name otherwise knife throws up
+        # when doing a knife cookbook share. So we're going to make a new tmp directory, create a subdirectory with the EXACT
+        # cookbook name, and then copy the cookbook contents into it. Yeah yeah, its pretty nasty, but blame Chef.
+        metadata_str = CapsuleCD::Chef::ChefHelper.read_repo_metadata(@source_git_local_path)
+        chef_metadata = CapsuleCD::Chef::ChefHelper.parse_metadata(metadata_str)
+
+        Dir.mktmpdir {|tmp_parent_path|
+          # create cookbook folder
+          tmp_local_path = File.join(tmp_parent_path, chef_metadata.name)
+          FileUtils.mkdir_p(tmp_local_path)
+          FileUtils.copy_entry(@source_git_local_path, tmp_local_path)
+
+          # write the knife.rb config jfile.
+          File.open(knife_path, 'w+') do |file|
+            file.write(<<-EOT.gsub(/^\s+/, '')
+            node_name "#{@config.chef_supermarket_username }" # Replace with the login name you use to login to the Supermarket.
+            client_key "#{pem_path}" # Define the path to wherever your client.pem file lives.  This is the key you generated when you signed up for a Chef account.
+            cookbook_path [ '#{tmp_parent_path}' ] # Directory where the cookbook you're uploading resides.
+            EOT
+            )
+          end
+
+          File.open(pem_path, 'w+') do |file|
+            file.write(@config.chef_supermarket_key)
+          end
+
+          command = "knife cookbook site share #{chef_metadata.name} #{@config.chef_supermarket_type} -c #{knife_path}"
+          Open3.popen3(command) do |_stdin, stdout, stderr, external|
+            { stdout: stdout, stderr: stderr }. each do |name, stream_buffer|
+              Thread.new do
+                until (line = stream_buffer.gets).nil?
+                  puts "#{name} -> #{line}"
+                end
+              end
+            end
+            # wait for process
+            external.join
+            unless external.value.success?
+              fail CapsuleCD::Error::ReleasePackageError, 'knife cookbook upload to supermarket failed'
+            end
+          end
+        }
+      end
+    end
+  end
+end

data/lib/capsulecd/chef/chef_helper.rb

@@ -0,0 +1,29 @@
+require 'ridley'
+require 'berkshelf'
+
+module CapsuleCD
+  module Chef
+    class ChefHelper
+      def self.read_repo_metadata(repo_path, metadata_filename = 'metadata.rb')
+        File.read("#{repo_path}/#{metadata_filename}")
+      end
+
+      def self.write_repo_metadata(repo_path, metadata_str, metadata_filename = 'metadata.rb')
+        File.open("#{repo_path}/#{metadata_filename}", 'w') { |file| file.write(metadata_str) }
+      end
+      def self.parse_metadata(metadata_str)
+        chef_metadata = Ridley::Chef::Cookbook::Metadata.new
+        chef_metadata.instance_eval(metadata_str)
+        chef_metadata
+      end
+
+      def self.parse_berksfile_lock_from_repo(repo_path, lockfile_filename = 'Berksfile.lock')
+        Berkshelf::Lockfile.from_file("#{repo_path}/#{lockfile_filename}")
+      end
+
+      def self.parse_berksfile_from_repo(repo_path, berksfile_filename = 'Berksfile')
+        Berkshelf::Lockfile.from_file("#{repo_path}/#{berksfile_filename}")
+      end
+    end
+  end
+end

data/lib/capsulecd/cli.rb

@@ -0,0 +1,64 @@
+require 'thor'
+require 'capsulecd'
+require 'pp'
+
+module CapsuleCD
+  # The command line interface for CapsuleCD.
+  class Cli < Thor
+
+    ##
+    # Run
+    ##
+    desc 'start', 'Start a new CapsuleCD package pipeline '
+    option :runner,
+           type: :string,
+           default: 'default', # can be :none, :circleci or :shippable (check the readme for why other hosted providers arn't supported.)
+           desc: 'The cloud CI runner that is running this PR. (Used to determine the Environmental Variables to parse)'
+
+    option :source,
+           type: :string,
+           default: 'default',
+           desc: 'The source for the code, used to determine which git endpoint to clone from, and create releases on'
+
+    option :package_type,
+           type: :string,
+           default: 'default',
+           desc: 'The type of package being built.'
+
+    option :dry_run,
+           type: :boolean,
+           default: false,
+           desc: 'Specifies that no changes should be pushed to source and no package will be released'
+
+    option :config_file,
+           type: :string,
+           default: nil,
+           desc: 'Specifies the location of the config file'
+
+    # Begin processing
+    def start
+      # parse runner from env
+      engine_opts = {}
+
+      engine_opts[:runner] = options[:runner].to_sym # TODO: we cant modify the hash sent by Thor, so we'll duplicate it
+      engine_opts[:source] = options[:source].to_sym
+      engine_opts[:package_type] = options[:package_type].to_sym
+      engine_opts[:dry_run] = options[:dry_run]
+      puts '###########################################################################################'
+      puts '# Configuration '
+      puts '###########################################################################################'
+      pp engine_opts
+
+      if engine_opts[:package_type] == :default
+
+        engine = CapsuleCD::Engine.new(engine_opts)
+      else
+        package_type = engine_opts[:package_type].to_s
+        require_relative "#{package_type}/#{package_type}_engine"
+        engine = CapsuleCD.const_get(package_type.capitalize).const_get("#{package_type.capitalize}Engine").new(engine_opts)
+      end
+
+      engine.start
+    end
+  end
+end

data/lib/capsulecd/error.rb

@@ -0,0 +1,51 @@
+module CapsuleCD
+  # The collection of Minimart specific errors.
+  module Error
+    class BaseError < StandardError; end
+
+    # Raised when the config file specifies a hook/override for a step when the type is :repo
+    class EngineTransformUnavailableStep < BaseError; end
+
+    # Raised when the source is not specified
+    class SourceUnspecifiedError < BaseError; end
+
+    # Raised when capsule cannot create an authenticated client for the source.
+    class SourceAuthenticationFailed < BaseError; end
+
+    # Raised when there is an error parsing the repo payload format.
+    class SourcePayloadFormatError < BaseError; end
+
+    # Raised when a source payload is unsupported/action is invalid
+    class SourcePayloadUnsupported < BaseError; end
+
+    # Raised when the user who started the packaging is unauthorized (non-collaborator)
+    class SourceUnauthorizedUser < BaseError; end
+
+    # Raised when the package is missing certain required files (ie metadata.rb, package.json, setup.py, etc)
+    class BuildPackageInvalid < BaseError; end
+
+    # Raised when the source could not be compiled or build for any reason
+    class BuildPackageFailed < BaseError; end
+
+    # Raised when package dependencies fail to install correctly.
+    class TestDependenciesError < BaseError; end
+
+    # Raised when the package test runner fails
+    class TestRunnerError < BaseError; end
+
+    # Raised when credentials required to upload/deploy new package are missing.
+    class ReleaseCredentialsMissing < BaseError; end
+
+    # Raised when an error occurs while uploading package.
+    class ReleasePackageError < BaseError; end
+
+    # Gracefully handle any errors raised by CapsuleCD, and exit with a failure
+    # status code.
+    # @param [CapsuleCD::Error::BaseError] ex
+    def self.handle_exception(ex)
+      puts ex.message
+      # Configuration.output.puts_red(ex.message)
+      exit false
+    end
+  end
+end