pipedream 0.1.0 → 0.4.0

data/lib/pipedream/dsl/webhook.rb

@@ -0,0 +1,27 @@
+module Pipedream::Dsl
+  module Webhook
+    include Ssm
+
+    # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-codepipeline-webhook.html
+    PROPERTIES = %w[
+      authentication
+      authentication_configuration
+      filters
+      name
+      register_with_third_party
+      target_action
+      target_pipeline
+      target_pipeline_version
+    ]
+    PROPERTIES.each do |prop|
+      define_method(prop) do |v|
+        @properties[prop.to_sym] = v
+      end
+    end
+
+    def secret_token(v)
+      @secret_token = v
+    end
+    alias_method :github_token, :secret_token
+  end
+end

data/lib/pipedream/evaluate.rb

@@ -0,0 +1,47 @@
+module Pipedream
+  module Evaluate
+    def evaluate(path)
+      source_code = IO.read(path)
+      begin
+        instance_eval(source_code, path)
+      rescue Exception => e
+        if e.class == SystemExit # allow exit to happen normally
+          raise
+        else
+          task_definition_error(e)
+          puts "\nFull error:"
+          raise
+        end
+      end
+    end
+
+  private
+    # Prints out a user friendly task_definition error message
+    def task_definition_error(e)
+      error_info = e.backtrace.first
+      path, line_no, _ = error_info.split(':')
+      line_no = line_no.to_i
+      puts "Error evaluating #{path}:".color(:red)
+      puts e.message
+      puts "Here's the line in #{path} with the error:\n\n"
+
+      contents = IO.read(path)
+      content_lines = contents.split("\n")
+      context = 5 # lines of context
+      top, bottom = [line_no-context-1, 0].max, line_no+context-1
+      spacing = content_lines.size.to_s.size
+      content_lines[top..bottom].each_with_index do |line_content, index|
+        line_number = top+index+1
+        if line_number == line_no
+          printf("%#{spacing}d %s\n".color(:red), line_number, line_content)
+        else
+          printf("%#{spacing}d %s\n", line_number, line_content)
+        end
+      end
+    end
+
+    def lookup_codepipeline_file(name)
+      [".pipedream", @options[:type], name].compact.join("/")
+    end
+  end
+end

data/lib/pipedream/help/completion.md

@@ -0,0 +1,22 @@
+Example:
+
+    pipe completion
+
+Prints words for TAB auto-completion.
+
+Examples:
+
+    pipe completion
+    pipe completion hello
+    pipe completion hello name
+
+To enable, TAB auto-completion add the following to your profile:
+
+    eval $(pipe completion_script)
+
+Auto-completion example usage:
+
+    pipe [TAB]
+    pipe hello [TAB]
+    pipe hello name [TAB]
+    pipe hello name --[TAB]

data/lib/pipedream/help/completion_script.md

@@ -0,0 +1,3 @@
+To use, add the following to your `~/.bashrc` or `~/.profile`
+
+    eval $(pipe completion_script)

data/lib/pipedream/help/deploy.md

@@ -0,0 +1,54 @@
+Examples:
+
+    pipe deploy
+    pipe deploy demo # explicitly specify pipeline name
+    pipe deploy demo -b mybranch # specify git branch
+
+The pipeline is generated from the DSL and created with CloudFormation. The files that the DSL evaluates are in the `.pipedream` folder:
+
+    .pipedream/pipeline.rb
+    .pipedream/role.rb
+    .pipedream/schedule.rb
+    .pipedream/webhook.rb
+
+To create the CodePipeline pipeline, you run:
+
+    pipe deploy
+
+You'll see output that looks something like this:
+
+    $ pipe deploy
+    Generated CloudFormation template at /tmp/codepipeline.yml
+    Deploying stack demo-pipe with CodePipeline project demo
+    Creating stack demo-pipe. Check CloudFormation console for status.
+    Stack name demo-pipe status CREATE_IN_PROGRESS
+    Here's the CloudFormation url to check for more details https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks
+    Waiting for stack to complete
+    04:14:03AM CREATE_IN_PROGRESS AWS::CloudFormation::Stack demo-pipe User Initiated
+    04:14:06AM CREATE_IN_PROGRESS AWS::IAM::Role IamRole
+    04:14:07AM CREATE_IN_PROGRESS AWS::IAM::Role IamRole Resource creation Initiated
+    04:14:25AM CREATE_COMPLETE AWS::IAM::Role IamRole
+    04:14:28AM CREATE_IN_PROGRESS AWS::CodePipeline::Pipeline Pipeline
+    04:14:29AM CREATE_IN_PROGRESS AWS::CodePipeline::Pipeline Pipeline Resource creation Initiated
+    04:14:29AM CREATE_COMPLETE AWS::CodePipeline::Pipeline Pipeline
+    04:14:31AM CREATE_IN_PROGRESS AWS::CodePipeline::Webhook Webhook
+    04:14:33AM CREATE_IN_PROGRESS AWS::CodePipeline::Webhook Webhook Resource creation Initiated
+    04:14:33AM CREATE_COMPLETE AWS::CodePipeline::Webhook Webhook
+    04:14:35AM CREATE_COMPLETE AWS::CloudFormation::Stack demo-pipe
+    Stack success status: CREATE_COMPLETE
+    Time took for stack deployment: 35s.
+    $
+
+## Explicit Pipeline Name
+
+By default, the pipeline name is inferred and is the parent folder that you are within.  You can explicitly specify the pipeline name as the first CLI argument:
+
+    pipe deploy my-pipeline
+
+## Specify Git Branch
+
+It is useful to build pipelines with different source git branches. You can pass a `--branch` option to the `pipe deploy` command. The cli `—-branch` option always takes the highest precedence. Example:
+
+    pipe deploy my-pipeline --branch my-branch
+
+Note: When you specify a branch pipedream actually first updates the pipeline before starting the pipeline execution. This is done because CodePipeline does not natively support specifying the branch. It is discussed more here: [Using Different Branches]({% link _docs/examples/different-branches.md %}).

data/lib/pipedream/help/start.md

@@ -0,0 +1,20 @@
+You can start a pipeline with the `pipe start` command. Here's an example:
+
+    $ pipe start
+    Pipeline started: demo
+    Please check the CodePipeline console for the status.
+    CodePipeline Console: https://us-west-2.console.aws.amazon.com/codesuite/codepipeline/pipelines/demo/view
+    Pipeline cli: aws codepipeline get-pipeline-execution --pipeline-execution-id 02579d64-9271-4edc-aa45-bc9629d732bb --pipeline-name demo
+    $
+
+## Specifying Code Branch
+
+If you would like start a build using a specific code branch you can use the `--branch` or `-b` option.  Example:
+
+    pipe start -b feature-branch
+
+## AWS CLI Equivalent
+
+The `pipe start` command is a simple wrapper to the AWS API with the ruby sdk. You can also start pipelines with the `aws codepipeline` cli.  Here's the equivalent CLI command:
+
+    aws codepipeline start-pipeline-execution --name demo

data/lib/pipedream/help.rb

@@ -0,0 +1,9 @@
+module Pipedream::Help
+  class << self
+    def text(namespaced_command)
+      path = namespaced_command.to_s.gsub(':','/')
+      path = File.expand_path("../help/#{path}.md", __FILE__)
+      IO.read(path) if File.exist?(path)
+    end
+  end
+end

data/lib/pipedream/init.rb

@@ -0,0 +1,68 @@
+module Pipedream
+  class Init < Sequence
+    # Ugly, this is how I can get the options from to match with this Thor::Group
+    def self.cli_options
+      [
+        [:name, desc: "CodePipeline project name."],
+        [:mode, desc: "Modes: light or full", default: "light" ],
+        [:force, type: :boolean, desc: "Bypass overwrite are you sure prompt for existing files."],
+        [:template, desc: "Custom template to use."],
+        [:template_mode, desc: "Template mode: replace or additive."],
+      ]
+    end
+    cli_options.each { |o| class_option(*o) }
+
+    def setup_template_repo
+      return unless @options[:template]&.include?('/')
+
+      sync_template_repo
+    end
+
+    def set_source_path
+      return unless @options[:template]
+
+      custom_template = "#{ENV['HOME']}/.pipedream/templates/#{full_repo_name}"
+
+      if @options[:template_mode] == "replace" # replace the template entirely
+        override_source_paths(custom_template)
+      else # additive: modify on top of default template
+        default_template = File.expand_path("../../template", __FILE__)
+        override_source_paths([custom_template, default_template])
+      end
+    end
+
+    def copy_project
+      puts "Initialize codepipeline project in .pipedream"
+
+      excludes = %w[.git]
+      if @options[:mode] == "light"
+        excludes += %w[
+          settings.yml
+          sns.rb
+        ]
+      end
+      pattern = Regexp.new(excludes.join('|'))
+
+      if @options[:template]
+        directory ".", ".pipedream", exclude_pattern: pattern
+      else
+        directory ".", exclude_pattern: pattern
+      end
+    end
+
+  private
+    def project_name
+      inferred_name = File.basename(Dir.pwd).gsub('_','-').gsub(/[^0-9a-zA-Z,-]/, '')
+      @options[:name] || inferred_name
+    end
+
+    def project_github_repo
+      default = "user/repo"
+      return default unless File.exist?(".git/config") && git_installed?
+
+      url = `git config --get remote.origin.url`.strip
+      repo = Dsl::Pipeline::Github.extract_repo_source(url)
+      repo == '' ? default : repo
+    end
+  end
+end

data/lib/pipedream/pipeline/s3_bucket.rb

@@ -0,0 +1,88 @@
+require "aws-sdk-s3"
+
+class Pipedream::Pipeline
+  class S3Bucket
+    extend Memoist
+    include Pipedream::AwsServices
+
+    class << self
+      extend Memoist
+      def name
+        new.name
+      end
+      memoize :name
+    end
+
+    def name
+      ensure_exists(bucket_name)
+      bucket_name
+    end
+    memoize :name
+
+    def bucket_name
+      "codepipeline-#{aws.region}-#{aws.account}"
+    end
+
+    def ensure_exists(name)
+      return if exists?(name) || ENV['TEST']
+      s3.create_bucket(bucket: name)
+      policy = {
+        "Version": "2012-10-17",
+        "Id": "SSEAndSSLPolicy",
+        "Statement": [
+          {
+            "Sid": "DenyUnEncryptedObjectUploads",
+            "Effect": "Deny",
+            "Principal": "*",
+            "Action": "s3:PutObject",
+            "Resource": "arn:aws:s3:::#{name}/*",
+            "Condition": {
+              "StringNotEquals": {
+                "s3:x-amz-server-side-encryption": "aws:kms"
+              }
+            }
+          },
+          {
+              "Sid": "DenyInsecureConnections",
+              "Effect": "Deny",
+              "Principal": "*",
+              "Action": "s3:*",
+              "Resource": "arn:aws:s3:::#{name}/*",
+              "Condition": {
+                  "Bool": {
+                      "aws:SecureTransport": "false"
+                  }
+              }
+          }
+        ]
+      }
+      s3.put_bucket_policy(
+        bucket: name,
+        policy: JSON.dump(policy),
+      )
+    rescue Aws::S3::Errors::BucketAlreadyExists => e
+      puts "ERROR #{e.class}: #{e.message}".color(:red)
+      puts "Bucket name: #{name}"
+      exit 1
+    end
+
+    def exists?(name)
+      begin
+        s3.head_bucket(bucket: name)
+        true
+      rescue Aws::S3::Errors::BucketAlreadyOwnedByYou, Aws::S3::Errors::Http301Error
+        # These exceptions indicate bucket already exists
+        # Aws::S3::Errors::Http301Error could be inaccurate but compromising for simplicity
+        true
+      rescue
+        false
+      end
+    end
+
+  private
+    def aws
+      AwsData.new
+    end
+    memoize :aws
+  end
+end

data/lib/pipedream/pipeline.rb

@@ -0,0 +1,61 @@
+module Pipedream
+  class Pipeline
+    extend Memoist
+    include Dsl::Pipeline
+    include Evaluate
+
+    def initialize(options={})
+      @options = options
+      @pipeline_path = options[:pipeline_path] || get_pipeline_path
+      @properties = default_properties # defaults make pipeline.rb simpler
+      @stages = []
+    end
+
+    def run
+      evaluate(@pipeline_path)
+      @properties[:stages] ||= @stages
+      set_source_branch!
+
+      resource = {
+        pipeline: {
+          type: "AWS::CodePipeline::Pipeline",
+          properties: @properties
+        }
+      }
+      CfnCamelizer.transform(resource)
+    end
+
+    def default_properties
+      {
+        name: @options[:full_pipeline_name],
+        role_arn: { "Fn::GetAtt": "IamRole.Arn" },
+        artifact_store: {
+          type: "S3",
+          location: s3_bucket, # auto creates s3 bucket
+        }
+      }
+    end
+
+    # cli branch option always takes highest precedence
+    def set_source_branch!
+      return unless @options[:branch]
+
+      source_stage = @properties[:stages].first
+      action = source_stage[:actions].first
+      action[:configuration][:branch] = @options[:branch]
+    end
+
+    def exist?
+      File.exist?(@pipeline_path)
+    end
+
+    def s3_bucket
+      S3Bucket.name
+    end
+
+  private
+    def get_pipeline_path
+      lookup_codepipeline_file "pipeline.rb"
+    end
+  end
+end

data/lib/pipedream/role.rb

@@ -0,0 +1,181 @@
+require "yaml"
+
+module Pipedream
+  class Role
+    include Pipedream::Dsl::Role
+    include Evaluate
+
+    def initialize(options={})
+      @options = options
+      @role_path = options[:role_path] || get_role_path
+      @properties = default_properties
+    end
+
+    def run
+      evaluate(@role_path) if File.exist?(@role_path)
+      @properties[:policies] = [{
+        policy_name: "CodePipelineAccess",
+        policy_document: {
+          version: "2012-10-17",
+          statement: derived_iam_statements
+        }
+      }]
+
+      @properties[:managed_policy_arns] = @managed_policy_arns if @managed_policy_arns && !@managed_policy_arns.empty?
+
+      resource = {
+        logical_id => {
+          type: "AWS::IAM::Role",
+          properties: @properties
+        }
+      }
+      CfnCamelizer.transform(resource)
+    end
+
+    def logical_id
+      "IamRole"
+    end
+
+  private
+    def get_role_path
+      lookup_codepipeline_file("role.rb")
+    end
+
+    def default_properties
+      {
+        assume_role_policy_document: {
+          statement: [{
+            action: ["sts:AssumeRole"],
+            effect: "Allow",
+            principal: {
+              service: principal_services
+            }
+          }],
+          version: "2012-10-17"
+        },
+        path: "/"
+      }
+    end
+
+    def principal_services
+      ["codepipeline.amazonaws.com"]
+    end
+
+    def derived_iam_statements
+      @iam_statements || default_iam_statements
+    end
+
+    def default_iam_statements
+      # Based on the one created by CodePipeline Console
+      [{
+        "action"=>["iam:PassRole"],
+        "resource"=>"*",
+        "effect"=>"Allow",
+        "condition"=>
+          {"string_equals_if_exists"=>
+            {"iam:passed_to_service"=>
+              ["cloudformation.amazonaws.com",
+               "elasticbeanstalk.amazonaws.com",
+               "ec2.amazonaws.com",
+               "ecs-tasks.amazonaws.com"]
+            }
+          }
+      },{
+        "action"=>
+          ["codecommit:CancelUploadArchive",
+           "codecommit:GetBranch",
+           "codecommit:GetCommit",
+           "codecommit:GetUploadArchiveStatus",
+           "codecommit:UploadArchive"],
+          "resource"=>"*",
+          "effect"=>"Allow"
+      },{
+        "action"=>
+         ["codedeploy:CreateDeployment",
+          "codedeploy:GetApplication",
+          "codedeploy:GetApplicationRevision",
+          "codedeploy:GetDeployment",
+          "codedeploy:GetDeploymentConfig",
+          "codedeploy:RegisterApplicationRevision"],
+          "resource"=>"*",
+          "effect"=>"Allow"
+      },{
+        "action"=>
+          ["elasticbeanstalk:*",
+           "ec2:*",
+           "elasticloadbalancing:*",
+           "autoscaling:*",
+           "cloudwatch:*",
+           "s3:*",
+           "sns:*",
+           "cloudformation:*",
+           "rds:*",
+           "sqs:*",
+           "ecs:*"],
+        "resource"=>"*",
+        "effect"=>"Allow"
+      },{
+        "action"=>["lambda:InvokeFunction", "lambda:ListFunctions"],
+        "resource"=>"*",
+        "effect"=>"Allow"
+      },{
+        "action"=>
+           ["opsworks:CreateDeployment",
+            "opsworks:DescribeApps",
+            "opsworks:DescribeCommands",
+            "opsworks:DescribeDeployments",
+            "opsworks:DescribeInstances",
+            "opsworks:DescribeStacks",
+            "opsworks:UpdateApp",
+            "opsworks:UpdateStack"],
+          "resource"=>"*",
+          "effect"=>"Allow"
+      },{
+        "action"=>
+         ["cloudformation:CreateStack",
+          "cloudformation:DeleteStack",
+          "cloudformation:DescribeStacks",
+          "cloudformation:UpdateStack",
+          "cloudformation:CreateChangeSet",
+          "cloudformation:DeleteChangeSet",
+          "cloudformation:DescribeChangeSet",
+          "cloudformation:ExecuteChangeSet",
+          "cloudformation:SetStackPolicy",
+          "cloudformation:ValidateTemplate"],
+        "resource"=>"*",
+        "effect"=>"Allow"
+      },{
+        "action"=>["codebuild:BatchGetBuilds", "codebuild:StartBuild"],
+        "resource"=>"*",
+        "effect"=>"Allow"
+      },{
+        "action"=>
+          ["devicefarm:ListProjects",
+           "devicefarm:ListDevicePools",
+           "devicefarm:GetRun",
+           "devicefarm:GetUpload",
+           "devicefarm:CreateUpload",
+           "devicefarm:ScheduleRun"],
+          "resource"=>"*",
+          "effect"=>"Allow",
+      },{
+        "action"=>
+          ["servicecatalog:ListProvisioningArtifacts",
+           "servicecatalog:CreateProvisioningArtifact",
+           "servicecatalog:DescribeProvisioningArtifact",
+           "servicecatalog:DeleteProvisioningArtifact",
+           "servicecatalog:UpdateProduct"],
+        "resource"=>"*",
+        "effect"=>"Allow",
+      },{
+        "action"=>["cloudformation:ValidateTemplate"],
+        "resource"=>"*",
+        "effect"=>"Allow",
+      },{
+        "action"=>["ecr:DescribeImages"],
+        "resource"=>"*",
+        "effect"=>"Allow",
+      }]
+    end
+  end
+end

data/lib/pipedream/schedule.rb

@@ -0,0 +1,99 @@
+module Pipedream
+  class Schedule
+    include Pipedream::Dsl::Schedule
+    include Evaluate
+
+    def initialize(options={})
+      @options = options
+      @schedule_path = options[:schedule_path] || get_schedule_path
+      @properties = default_properties
+    end
+
+    def run
+      return unless File.exist?(@schedule_path)
+
+      old_properties = @properties.clone
+      evaluate(@schedule_path)
+
+      @properties[:schedule_expression] = @schedule_expression if @schedule_expression
+      set_rule_event! if @rule_event_props
+      return if old_properties == @properties # empty schedule.rb file
+
+      resource = {
+        events_rule: {
+          type: "AWS::Events::Rule",
+          properties: @properties
+        },
+        events_rule_role: events_rule_role,
+      }
+      CfnCamelizer.transform(resource)
+    end
+
+    def set_rule_event!
+      props = @rule_event_props
+      if props.key?(:detail)
+        description = props.key?(:description) ? props.delete(:description) : rule_description
+        rule_props = { event_pattern: props, description: description }
+      else # if props.key?(:event_pattern)
+        props[:description] ||= rule_description
+        rule_props = props
+      end
+
+      @properties.merge!(rule_props)
+    end
+
+    def default_properties
+      description = "CodePipeline #{@options[:full_pipeline_name]}"
+      name = description.gsub(" ", "-").downcase
+      {
+        description: description,
+        # event_pattern: ,
+        name: name,
+        # schedule_expression: ,
+        state: "ENABLED",
+        targets: [{
+          arn: "arn:aws:codepipeline:#{aws.region}:#{aws.account}:#{@options[:full_pipeline_name]}",
+          role_arn: { "Fn::GetAtt": "EventsRuleRole.Arn" }, # required for specific CodePipeline target.
+          id: "CodePipelineTarget",
+        }]
+      }
+    end
+
+  private
+    def get_schedule_path
+      lookup_codepipeline_file("schedule.rb")
+    end
+
+    def events_rule_role
+      {
+        type: "AWS::IAM::Role",
+        properties: {
+          assume_role_policy_document: {
+            statement: [{
+              action: [ "sts:AssumeRole" ],
+              effect: "Allow",
+              principal: { service: [ "events.amazonaws.com" ] }
+            }],
+            version: "2012-10-17"
+          },
+          path: "/",
+          policies: [{
+            policy_name: "CodePipelineAccess",
+            policy_document: {
+              version: "2012-10-17",
+              statement: [{
+                action: "codepipeline:StartPipelineExecution",
+                effect: "Allow",
+                resource: "arn:aws:codepipeline:#{aws.region}:#{aws.account}:#{@options[:full_pipeline_name]}"
+              }]
+            }
+          }]
+        }
+      }
+    end
+
+    def aws
+      @aws ||= AwsData.new
+    end
+  end
+end