pipedream 0.1.0 → 0.4.0

data/lib/pipedream/sequence.rb

@@ -0,0 +1,66 @@
+require 'fileutils'
+require 'thor'
+
+module Pipedream
+  class Sequence < Thor::Group
+    include AwsServices
+    include Thor::Actions
+
+    add_runtime_options! # force, pretend, quiet, skip options
+      # https://github.com/erikhuda/thor/blob/master/lib/thor/actions.rb#L49
+
+    def self.source_paths
+      [File.expand_path("../../template", __FILE__)]
+    end
+
+  private
+    def override_source_paths(*paths)
+      # Using string with instance_eval because block doesnt have access to
+      # path at runtime.
+      self.class.instance_eval %{
+        def self.source_paths
+          #{paths.flatten.inspect}
+        end
+      }
+    end
+
+    def sync_template_repo
+      unless git_installed?
+        abort "Unable to detect git installation on your system.  Git needs to be installed in order to use the --template option."
+      end
+
+      template_path = "#{ENV['HOME']}/.pipedream/templates/#{full_repo_name}"
+      if File.exist?(template_path)
+        sh("cd #{template_path} && git pull")
+      else
+        FileUtils.mkdir_p(File.dirname(template_path))
+        sh("git clone #{repo_url} #{template_path}")
+      end
+    end
+
+    def full_repo_name
+      full_repo = options[:template].split("/")[-2..-1].join("/")
+      full_repo = full_repo.split(":").last
+      full_repo.sub(".git", "")
+    end
+
+    # normalize repo_url
+    def repo_url
+      template = options[:template]
+      if template.include?('github.com')
+        template # leave as is, user has provided full github url
+      else
+        "https://github.com/#{template}"
+      end
+    end
+
+    def git_installed?
+      system("type git > /dev/null")
+    end
+
+    def sh(command)
+      puts "=> #{command}"
+      system(command)
+    end
+  end
+end

data/lib/pipedream/setting.rb

@@ -0,0 +1,82 @@
+require 'yaml'
+require 'render_me_pretty'
+
+module Pipedream
+  class Setting
+    extend Memoist
+    def initialize(check_codepipeline_project=true)
+      @check_codepipeline_project = check_codepipeline_project
+    end
+
+    # data contains the settings.yml config.  The order or precedence for settings
+    # is the project ufo/settings.yml and then the ~/.pipedream/settings.yml.
+    def data
+      Pipedream.check_codepipeline_project! if @check_codepipeline_project
+      return {} unless File.exist?(project_settings_path)
+
+      # project based settings files
+      project = load_file(project_settings_path)
+
+      user_file = "#{ENV['HOME']}/.pipedream/settings.yml"
+      user = File.exist?(user_file) ? YAML.load_file(user_file) : {}
+
+      default_file = File.expand_path("default/settings.yml", __dir__)
+      default = load_file(default_file)
+
+      all_envs = default.deep_merge(user.deep_merge(project))
+      all_envs = merge_base(all_envs)
+      data = all_envs[pipe_env] || all_envs["base"] || {}
+      data.deep_symbolize_keys
+    end
+    memoize :data
+
+    # Resolves infinite problem since Pipedream.env can be determined from PIPE_ENV or settings.yml files.
+    # When ufo is determined from settings it should not called Pipedream.env since that in turn calls
+    # Settings.new.data which can then cause an infinite loop.
+    def pipe_env
+      path = "#{cb_root}/.pipedream/settings.yml"
+      if File.exist?(path)
+        settings = YAML.load_file(path)
+        env = settings.find do |_env, section|
+          section ||= {}
+          ENV['AWS_PROFILE'] && ENV['AWS_PROFILE'] == section['aws_profile']
+        end
+      end
+
+      pipe_env = env.first if env
+      pipe_env = ENV['PIPE_ENV'] if ENV['PIPE_ENV'] # highest precedence
+      pipe_env || 'development'
+    end
+
+  private
+    def load_file(path)
+      return Hash.new({}) unless File.exist?(path)
+
+      content = RenderMePretty.result(path)
+      data = YAML.load(content)
+      # If key is is accidentally set to nil it screws up the merge_base later.
+      # So ensure that all keys with nil value are set to {}
+      data.each do |env, _setting|
+        data[env] ||= {}
+      end
+      data
+    end
+
+    # automatically add base settings to the rest of the environments
+    def merge_base(all_envs)
+      base = all_envs["base"] || {}
+      all_envs.each do |env, settings|
+        all_envs[env] = base.merge(settings) unless env == "base"
+      end
+      all_envs
+    end
+
+    def project_settings_path
+      "#{cb_root}/.pipedream/settings.yml"
+    end
+
+    def cb_root
+      ENV["CODEPIPELINE_ROOT"] || Dir.pwd
+    end
+  end
+end

data/lib/pipedream/sns.rb

@@ -0,0 +1,43 @@
+module Pipedream
+  class Sns
+    include Pipedream::Dsl::Sns
+    include Evaluate
+
+    def initialize(options={})
+      @options = options
+      @sns_path = options[:sns_path] || get_sns_path
+      @properties = default_properties
+    end
+
+    def run
+      evaluate(@sns_path) if File.exist?(@sns_path)
+
+      resource = {
+        sns_topic: {
+          type: "AWS::SNS::Topic",
+          properties: @properties
+        }
+      }
+      CfnCamelizer.transform(resource)
+    end
+
+    def default_properties
+      display_name = "#{@options[:full_pipeline_name]} pipeline"
+      {
+        display_name: display_name,
+        # kms_master_key_id: "string",
+        # subscription: [{
+        #   endpoint: '',
+        #   protocol: ','
+        # }],
+        # topic_name: "string", # Not setting because update requires: Replacement. Dont want 2 pipelines to collide
+      }
+      # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-sns-subscription.html
+    end
+
+  private
+    def get_sns_path
+      lookup_codepipeline_file("sns.rb")
+    end
+  end
+end

data/lib/pipedream/stack.rb

@@ -0,0 +1,95 @@
+require "aws-sdk-cloudformation"
+
+module Pipedream
+  class Stack
+    include AwsServices
+
+    def initialize(options)
+      @options = options
+      @pipeline_name = @options[:pipeline_name] || inferred_pipeline_name
+      @stack_name = options[:stack_name] || inferred_stack_name(@pipeline_name)
+
+      @full_pipeline_name = pipeline_name_convention(@pipeline_name)
+      @template = {
+        "Description" => "CodePipeline Project: #{@full_pipeline_name}",
+        "Resources" => {}
+      }
+    end
+
+    def run
+      options = @options.merge(
+        pipeline_name: @pipeline_name,
+        full_pipeline_name: @full_pipeline_name,
+      )
+
+      pipeline_builder = Pipeline.new(options)
+      unless pipeline_builder.exist?
+        puts "ERROR: pipeline does not exist: #{pipeline_builder.pipeline_path}".color(:red)
+        exit 1
+        return
+      end
+      pipeline = pipeline_builder.run
+      @template["Resources"].merge!(pipeline)
+
+      if pipeline["Pipeline"]["Properties"]["RoleArn"] == {"Fn::GetAtt"=>"IamRole.Arn"}
+        role = Role.new(options).run
+        @template["Resources"].merge!(role)
+      end
+
+      if sns_topic?(pipeline)
+        role = Sns.new(options).run
+        @template["Resources"].merge!(role)
+      end
+
+      webhook = Webhook.new(options).run
+      @template["Resources"].merge!(webhook) if webhook
+
+      schedule = Schedule.new(options).run
+      @template["Resources"].merge!(schedule) if schedule
+
+      template_path = "/tmp/codepipeline.yml"
+      FileUtils.mkdir_p(File.dirname(template_path))
+      IO.write(template_path, YAML.dump(@template))
+      puts "Generated CloudFormation template at #{template_path.color(:green)}"
+      return if @options[:noop]
+      puts "Deploying stack #{@stack_name.color(:green)} with CodePipeline project #{@full_pipeline_name.color(:green)}"
+
+      begin
+        perform
+        url_info
+        return unless @options[:wait]
+        status.wait
+        exit 2 unless status.success?
+      rescue Aws::CloudFormation::Errors::ValidationError => e
+        if e.message.include?("No updates") # No updates are to be performed.
+          puts "WARN: #{e.message}".color(:yellow)
+        else
+          puts "ERROR ValidationError: #{e.message}".color(:red)
+          exit 1
+        end
+      end
+    end
+
+  private
+    def sns_topic?(template)
+      stages = template['Pipeline']['Properties']['Stages']
+      stages.detect do |stage|
+        stage['Actions'].detect do |action|
+          action['Configuration']['NotificationArn'] == {'Ref'=>'SnsTopic'}
+        end
+      end
+    end
+
+    def url_info
+      stack = cfn.describe_stacks(stack_name: @stack_name).stacks.first
+      region = `aws configure get region`.strip rescue "us-east-1"
+      url = "https://console.aws.amazon.com/cloudformation/home?region=#{region}#/stacks"
+      puts "Stack name #{@stack_name.color(:yellow)} status #{stack["stack_status"].color(:yellow)}"
+      puts "Here's the CloudFormation url to check for more details #{url}"
+    end
+
+    def status
+      @status ||= Cfn::Status.new(@stack_name)
+    end
+  end
+end

data/lib/pipedream/start.rb

@@ -0,0 +1,84 @@
+module Pipedream
+  class Start
+    extend Memoist
+    include AwsServices
+
+    def initialize(options)
+      @options = options
+      @pipeline_name = options[:pipeline_name] || inferred_pipeline_name
+      @full_pipeline_name = pipeline_name_convention(@pipeline_name)
+      @stack_name = options[:stack_name] || inferred_stack_name(@pipeline_name)
+    end
+
+    def run
+      check_pipeline_exists!
+      redeploy
+      resp = codepipeline.start_pipeline_execution(name: pipeline_name)
+      codepipeline_info(resp.pipeline_execution_id)
+    end
+
+    # Pipedreamline does not currently support specifying a different branch starting an execution.
+    # Workaround this limitation by updating the pipeline and then starting the execution.
+    def redeploy
+      return unless different_branch?
+      puts "Different branch detected."
+      puts "  Current pipeline branch: #{current_pipeline_branch}"
+      puts "  Requested branch: #{@options[:branch]}"
+      puts "Updating pipeline with new branch.".color(:green)
+      Deploy.new(@options).run
+    end
+
+    def different_branch?
+      return false unless @options[:branch]
+      current_pipeline_branch != @options[:branch]
+    end
+
+    # Actual branch on current pipeline
+    def current_pipeline_branch
+      resp = codepipeline.get_pipeline(name: pipeline_name)
+      source_stage = resp.pipeline.stages.find { |s| s.name == "Source" }
+      action = source_stage.actions.first
+      action.configuration['Branch']
+    end
+    memoize :current_pipeline_branch
+
+    def check_pipeline_exists!
+      pipeline_name
+    end
+
+    def pipeline_name
+      if pipeline_exists?(@full_pipeline_name)
+        @full_pipeline_name
+      elsif stack_exists?(@stack_name) # allow `cb start STACK_NAME` to work too
+        resp = cfn.describe_stack_resources(stack_name: @stack_name)
+        resource = resp.stack_resources.find do |r|
+          r.logical_resource_id == "CodePipeline"
+        end
+        resource.physical_resource_id # codepipeline project name
+      else
+        puts "ERROR: Unable to find the codepipeline project with either full_pipeline_name: #{@full_pipeline_name} or stack name: #{@stack_name}".color(:red)
+        exit 1
+      end
+    end
+    memoize :pipeline_name
+
+  private
+    def codepipeline_info(execution_id)
+      region = `aws configure get region`.strip rescue "us-east-1"
+      url = "https://#{region}.console.aws.amazon.com/codesuite/codepipeline/pipelines/#{pipeline_name}/view"
+      cli = "aws codepipeline get-pipeline-execution --pipeline-execution-id #{execution_id} --pipeline-name #{pipeline_name}"
+
+      puts "Pipeline started: #{pipeline_name}"
+      puts "Please check the CodePipeline console for the status."
+      puts "CodePipeline Console: #{url}"
+      puts "Pipeline cli: #{cli}"
+    end
+
+    def pipeline_exists?(name)
+      codepipeline.get_pipeline(name: name)
+      true
+    rescue Aws::CodePipeline::Errors::PipelineNotFoundException
+      false
+    end
+  end
+end

data/lib/pipedream/update.rb

@@ -0,0 +1,12 @@
+module Pipedream
+  class Update < Stack
+    def perform
+      cfn.update_stack(
+        stack_name: @stack_name,
+        template_body: YAML.dump(@template),
+        capabilities: ["CAPABILITY_IAM"]
+      )
+      puts "Updating stack #{@stack_name}. Check CloudFormation console for status."
+    end
+  end
+end

data/lib/pipedream/version.rb

@@ -1,3 +1,3 @@
 module Pipedream
-  VERSION = "0.1.0"
+  VERSION = "0.4.0"
 end

data/lib/pipedream/webhook.rb

@@ -0,0 +1,60 @@
+module Pipedream
+  class Webhook
+    include Pipedream::Dsl::Webhook
+    include Evaluate
+
+    def initialize(options={})
+      @options = options
+      @webhook_path = options[:webhook_path] || get_webhook_path
+      @properties = default_properties
+    end
+
+    def run
+      return unless File.exist?(@webhook_path)
+
+      old_properties = @properties.clone
+      evaluate(@webhook_path)
+      set_secret_token!
+      return if old_properties == @properties # empty webhook.rb file
+
+      resource = {
+        webhook: {
+          type: "AWS::CodePipeline::Webhook",
+          properties: @properties
+        }
+      }
+      CfnCamelizer.transform(resource)
+    end
+
+    def default_properties
+      {
+        authentication: 'GITHUB_HMAC', # GITHUB_HMAC, IP and UNAUTHENTICATED
+        authentication_configuration: {
+           secret_token: @secret_token,
+        },
+        filters: [{
+          json_path: "$.ref",
+          match_equals: "refs/heads/{Branch}",
+        }],
+        # name: '', # optional
+        register_with_third_party: 'true', # optional
+        target_action: 'Source',
+        target_pipeline: {ref: "Pipeline"},
+        target_pipeline_version: {"Fn::GetAtt": "Pipeline.Version"},
+      }
+    end
+
+    def set_secret_token!
+      @properties.merge!(
+        authentication_configuration: {
+          secret_token: @secret_token
+        }
+      )
+    end
+  private
+
+    def get_webhook_path
+      lookup_codepipeline_file("webhook.rb")
+    end
+  end
+end

data/lib/pipedream.rb

@@ -1,6 +1,23 @@
+$:.unshift(File.expand_path("../", __FILE__))
 require "pipedream/version"
+require "rainbow/ext/string"
+require "memoist"
+require "active_support/core_ext/hash"
+
+require "pipedream/autoloader"
+Pipedream::Autoloader.setup
+
+gem_root = File.dirname(__dir__)
+$:.unshift("#{gem_root}/vendor/aws_data/lib")
+require "aws_data"
+$:.unshift("#{gem_root}/vendor/cfn_camelizer/lib")
+require "cfn_camelizer"
+$:.unshift("#{gem_root}/vendor/cfn-status/lib")
+require "cfn/status"
 
 module Pipedream
   class Error < StandardError; end
-  # Your code goes here...
+  extend Core
 end
+
+Pipedream.set_aws_profile!

data/lib/template/.pipedream/pipeline.rb.tt

@@ -0,0 +1,40 @@
+stage "Source" do
+  github(
+    source: "<%= project_github_repo %>",
+    # branch: "master", # branch defaults to "master" or the `pipe deploy --branch` option
+    auth_token: ssm("/codepipeline/github/token") # example ssm name
+  )
+end
+
+# IMPORANT: A valid pipeline requires at least 2 stages befre you are able to run
+#
+#     pipe deploy
+#
+# Here are some possible examples below. If you need help creating a CodeBuild project one, check out
+# https://pipedream.run/docs/examples/codebuild-project/
+
+# stage "Build" do
+#   codebuild "demo"
+# end
+
+# stage "MoreBuilds" do
+#   codebuild "project1"
+#   codebuild "project2", "project3" # runs in parallel
+#   codebuild "project4"
+# end
+
+# stage "Approve" do
+#   # Existing SNS Topic
+#   # approve(
+#   #   notification_arn: "arn:aws:sns:us-west-2:536766270177:hello-topic",
+#   #   custom_data: "Approve deployment",
+#   # )
+#
+#   # OR
+#   # CodePipeline will create and managed the SNS Topoic
+#   approve("Approve deployment")
+# end
+
+# stage "Deploy" do
+#   codebuild "project5"
+# end

data/lib/template/.pipedream/schedule.rb

@@ -0,0 +1,3 @@
+# rate "1 day"
+# or
+# cron("0 10 * * ? *") # Run at 10:00 am (UTC) every day

data/lib/template/.pipedream/settings.yml

@@ -0,0 +1,9 @@
+base:
+  # stack_naming:
+  #   append_env: true # default false
+
+development:
+  # aws_profile: dev_profile
+
+production:
+  # aws_profile: prod_profile

data/lib/template/.pipedream/sns.rb

@@ -0,0 +1,14 @@
+# If user has specified their own existing SNS topic ARN, then this SNS Topic managed by the codepipeline
+# tool will not get created. Also, only gets created if there's an approval action in the pipeline.
+#
+# Example properties:
+#
+#   https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-sns-topic.html
+#
+# display_name "my display_name"
+# kms_master_key_id "String"
+# subscription([{
+#   endpoint: '',
+#   protocol: ','
+# }])
+# topic_name "string", # Recommend not setting because update requires: Replacement. Allow CloudFormation to set it so 2 pipelines dont have same SNS Topic name that collides

data/pipedream.gemspec

@@ -1,4 +1,5 @@
-lib = File.expand_path("lib", __dir__)
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "pipedream/version"
 
@@ -7,25 +8,35 @@ Gem::Specification.new do |spec|
   spec.version       = Pipedream::VERSION
   spec.authors       = ["Tung Nguyen"]
   spec.email         = ["tongueroo@gmail.com"]
-
-  spec.summary       = "Tool"
+  spec.summary       = "A beautiful and powerful DSL to create and manage AWS CodePipeline pipelines"
   spec.homepage      = "https://github.com/tongueroo/pipedream"
   spec.license       = "MIT"
 
-  spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = "https://github.com/tongueroo/pipedream"
-  spec.metadata["changelog_uri"] = "https://github.com/tongueroo/pipedream/blob/master/CHANGELOG.md"
-
-  # Specify which files should be added to the gem when it is released.
-  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  vendor_files       = Dir.glob("vendor/**/*")
+  gem_files          = `git -C "#{File.dirname(__FILE__)}" ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features|docs)/})
   end
+  spec.files         = gem_files + vendor_files
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 2.0"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_dependency "activesupport"
+  spec.add_dependency "aws-sdk-cloudformation"
+  spec.add_dependency "aws-sdk-codepipeline"
+  spec.add_dependency "aws-sdk-s3"
+  spec.add_dependency "aws-sdk-ssm"
+  spec.add_dependency "cfn_camelizer"
+  spec.add_dependency "memoist"
+  spec.add_dependency "rainbow"
+  spec.add_dependency "render_me_pretty"
+  spec.add_dependency "thor"
+  spec.add_dependency "zeitwerk"
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "byebug"
+  spec.add_development_dependency "cli_markdown"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
 end

data/vendor/aws_data/CHANGELOG.md

@@ -0,0 +1,7 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+This project *loosely tries* to adhere to [Semantic Versioning](http://semver.org/).
+
+## [0.1.0]
+- Initial release

data/vendor/aws_data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in aws_data.gemspec
+gemspec

data/vendor/aws_data/Gemfile.lock

@@ -0,0 +1,48 @@
+PATH
+  remote: .
+  specs:
+    aws_data (0.1.0)
+      aws-sdk-core
+      memoist
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-eventstream (1.0.3)
+    aws-partitions (1.176.0)
+    aws-sdk-core (3.56.0)
+      aws-eventstream (~> 1.0, >= 1.0.2)
+      aws-partitions (~> 1.0)
+      aws-sigv4 (~> 1.1)
+      jmespath (~> 1.0)
+    aws-sigv4 (1.1.0)
+      aws-eventstream (~> 1.0, >= 1.0.2)
+    diff-lcs (1.3)
+    jmespath (1.4.0)
+    memoist (0.16.0)
+    rake (10.5.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.1)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.4)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aws_data!
+  bundler (~> 2.0)
+  rake (~> 10.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   2.0.1