pipedream 0.1.0 → 0.4.0

data/lib/pipedream/command.rb

@@ -0,0 +1,82 @@
+require "thor"
+
+# Override thor's long_desc identation behavior
+# https://github.com/erikhuda/thor/issues/398
+class Thor
+  module Shell
+    class Basic
+      def print_wrapped(message, options = {})
+        message = "\n#{message}" unless message[0] == "\n"
+        stdout.puts message
+      end
+    end
+  end
+end
+
+module Pipedream
+  class Command < Thor
+    class << self
+      def dispatch(m, args, options, config)
+        # Allow calling for help via:
+        #   codepipe command help
+        #   codepipe command -h
+        #   codepipe command --help
+        #   codepipe command -D
+        #
+        # as well thor's normal way:
+        #
+        #   codepipe help command
+        help_flags = Thor::HELP_MAPPINGS + ["help"]
+        if args.length > 1 && !(args & help_flags).empty?
+          args -= help_flags
+          args.insert(-2, "help")
+        end
+
+        #   codepipe version
+        #   codepipe --version
+        #   codepipe -v
+        version_flags = ["--version", "-v"]
+        if args.length == 1 && !(args & version_flags).empty?
+          args = ["version"]
+        end
+
+        super
+      end
+
+      # Override command_help to include the description at the top of the
+      # long_description.
+      def command_help(shell, command_name)
+        meth = normalize_command_name(command_name)
+        command = all_commands[meth]
+        alter_command_description(command)
+        super
+      end
+
+      def alter_command_description(command)
+        return unless command
+
+        # Add description to beginning of long_description
+        long_desc = if command.long_description
+            "#{command.description}\n\n#{command.long_description}"
+          else
+            command.description
+          end
+
+        # add reference url to end of the long_description
+        unless website.empty?
+          full_command = [command.ancestor_name, command.name].compact.join('-')
+          url = "#{website}/reference/codepipe-#{full_command}"
+          long_desc += "\n\nHelp also available at: #{url}"
+        end
+
+        command.long_description = long_desc
+      end
+      private :alter_command_description
+
+      # meant to be overriden
+      def website
+        ""
+      end
+    end
+  end
+end

data/lib/pipedream/completer/script.rb

@@ -0,0 +1,6 @@
+class Pipedream::Completer::Script
+  def self.generate
+    bash_script = File.expand_path("script.sh", File.dirname(__FILE__))
+    puts "source #{bash_script}"
+  end
+end

data/lib/pipedream/completer/script.sh

@@ -0,0 +1,10 @@
+_codepipe() {
+  COMPREPLY=()
+  local word="${COMP_WORDS[COMP_CWORD]}"
+  local words=("${COMP_WORDS[@]}")
+  unset words[0]
+  local completion=$(codepipe completion ${words[@]})
+  COMPREPLY=( $(compgen -W "$completion" -- "$word") )
+}
+
+complete -F _codepipe codepipe

data/lib/pipedream/completer.rb

@@ -0,0 +1,159 @@
+=begin
+Code Explanation:
+
+There are 3 types of things to auto-complete:
+
+  1. command: the command itself
+  2. parameters: command parameters.
+  3. options: command options
+
+Here's an example:
+
+  mycli hello name --from me
+
+  * command: hello
+  * parameters: name
+  * option: --from
+
+When command parameters are done processing, the remaining completion words will be options.  We can tell that the command params are completed based on the method arity.
+
+## Arity
+
+For example, say you had a method for a CLI command with the following form:
+
+  ufo scale service count --cluster development
+
+It's equivalent ruby method:
+
+  scale(service, count) = has an arity of 2
+
+So typing:
+
+  ufo scale service count [TAB] # there are 3 parameters including the "scale" command according to Thor's CLI processing.
+
+So the completion should only show options, something like this:
+
+  --noop --verbose --cluster
+
+## Splat Arguments
+
+When the ruby method has a splat argument, it's arity is negative.  Here are some example methods and their arities.
+
+   ship(service) = 1
+   scale(service, count) = 2
+   ships(*services) = -1
+   foo(example, *rest) = -2
+
+Fortunately, negative and positive arity values are processed the same way. So we take simply take the absolute value of the arity and process it the same.
+
+Here are some test cases, hit TAB after typing the command:
+
+  codepipe completion
+  codepipe completion hello
+  codepipe completion hello name
+  codepipe completion hello name --
+  codepipe completion hello name --noop
+
+  codepipe completion
+  codepipe completion sub:goodbye
+  codepipe completion sub:goodbye name
+
+## Subcommands and Thor::Group Registered Commands
+
+Sometimes the commands are not simple thor commands but are subcommands or Thor::Group commands. A good specific example is the ufo tool.
+
+  * regular command: ufo ship
+  * subcommand: ufo docker
+  * Thor::Group command: ufo init
+
+Auto-completion accounts for each of these type of commands.
+=end
+module Pipedream
+  class Completer
+    def initialize(command_class, *params)
+      @params = params
+      @current_command = @params[0]
+      @command_class = command_class # CLI initiall
+    end
+
+    def run
+      if subcommand?(@current_command)
+        subcommand_class = @command_class.subcommand_classes[@current_command]
+        @params.shift # destructive
+        Completer.new(subcommand_class, *@params).run # recursively use subcommand
+        return
+      end
+
+      # full command has been found!
+      unless found?(@current_command)
+        puts all_commands
+        return
+      end
+
+      # will only get to here if command aws found (above)
+      arity = @command_class.instance_method(@current_command).arity.abs
+      if @params.size > arity or thor_group_command?
+        puts options_completion
+      else
+        puts params_completion
+      end
+    end
+
+    def subcommand?(command)
+      @command_class.subcommands.include?(command)
+    end
+
+    # hacky way to detect that command is a registered Thor::Group command
+    def thor_group_command?
+      command_params(raw=true) == [[:rest, :args]]
+    end
+
+    def found?(command)
+      public_methods = @command_class.public_instance_methods(false)
+      command && public_methods.include?(command.to_sym)
+    end
+
+    # all top-level commands
+    def all_commands
+      commands = @command_class.all_commands.reject do |k,v|
+        v.is_a?(Thor::HiddenCommand)
+      end
+      commands.keys
+    end
+
+    def command_params(raw=false)
+      params = @command_class.instance_method(@current_command).parameters
+      # Example:
+      # >> Sub.instance_method(:goodbye).parameters
+      # => [[:req, :name]]
+      # >>
+      raw ? params : params.map!(&:last)
+    end
+
+    def params_completion
+      offset = @params.size - 1
+      offset_params = command_params[offset..-1]
+      command_params[offset..-1].first
+    end
+
+    def options_completion
+      used = ARGV.select { |a| a.include?('--') } # so we can remove used options
+
+      method_options = @command_class.all_commands[@current_command].options.keys
+      class_options = @command_class.class_options.keys
+
+      all_options = method_options + class_options + ['help']
+
+      all_options.map! { |o| "--#{o.to_s.gsub('_','-')}" }
+      filtered_options = all_options - used
+      filtered_options.uniq
+    end
+
+    # Useful for debugging. Using puts messes up completion.
+    def log(msg)
+      File.open("/tmp/complete.log", "a") do |file|
+        file.puts(msg)
+      end
+    end
+  end
+end

data/lib/pipedream/core.rb

@@ -0,0 +1,63 @@
+require 'pathname'
+require 'yaml'
+require 'active_support/core_ext/string'
+
+module Pipedream
+  module Core
+    extend Memoist
+
+    def root
+      path = ENV['PIPE_ROOT'] || '.'
+      Pathname.new(path)
+    end
+
+    def env
+      # 2-way binding
+      pipe_env = env_from_profile || 'development'
+      pipe_env = ENV['PIPE_ENV'] if ENV['PIPE_ENV'] # highest precedence
+      ActiveSupport::StringInquirer.new(pipe_env)
+    end
+    memoize :env
+
+    def env_extra
+      env_extra = ENV['PIPE_ENV_EXTRA'] if ENV['PIPE_ENV_EXTRA'] # highest precedence
+      return if env_extra&.empty?
+      env_extra
+    end
+    memoize :env_extra
+
+    # Overrides AWS_PROFILE based on the Pipedream.env if set in configs/settings.yml
+    # 2-way binding.
+    def set_aws_profile!
+      return if ENV['TEST']
+      return unless File.exist?("#{Pipedream.root}/.pipedream/settings.yml") # for rake docs
+      return unless settings # Only load if within Pipedream project and there's a settings.yml
+
+      data = settings || {}
+      if data[:aws_profile]
+        puts "Using AWS_PROFILE=#{data[:aws_profile]} from PIPE_ENV=#{Pipedream.env} in config/settings.yml"
+        ENV['AWS_PROFILE'] = data[:aws_profile]
+      end
+    end
+
+    def settings
+      Setting.new.data
+    end
+    memoize :settings
+
+    def check_codepipeline_project!
+      check_path = "#{Pipedream.root}/.pipedream"
+      unless File.exist?(check_path)
+        puts "ERROR: No .pipedream folder found.  Are you sure you are in a project with codepipeline setup?".color(:red)
+        puts "Current directory: #{Dir.pwd}"
+        puts "If you want to set up codepipeline for this prjoect, please create a settings file via: codepipeline init"
+        exit 1 unless ENV['TEST']
+      end
+    end
+
+  private
+    def env_from_profile
+      Pipedream::Setting.new.pipe_env
+    end
+  end
+end

data/lib/pipedream/create.rb

@@ -0,0 +1,12 @@
+module Pipedream
+  class Create < Stack
+    def perform
+      cfn.create_stack(
+        stack_name: @stack_name,
+        template_body: YAML.dump(@template),
+        capabilities: ["CAPABILITY_IAM"]
+      )
+      puts "Creating stack #{@stack_name}. Check CloudFormation console for status."
+    end
+  end
+end

data/lib/pipedream/delete.rb

@@ -0,0 +1,27 @@
+module Pipedream
+  class Delete
+    include AwsServices
+
+    def initialize(options)
+      @options = options
+      @pipeline_name = options[:pipeline_name] || inferred_pipeline_name
+      @stack_name = options[:stack_name] || inferred_stack_name(@pipeline_name)
+    end
+
+    def run
+      message = "Deleted #{@stack_name} stack."
+      if @options[:noop]
+        puts "NOOP #{message}"
+      else
+        are_you_sure?(@stack_name, :delete)
+
+        if stack_exists?(@stack_name)
+          cfn.delete_stack(stack_name: @stack_name)
+          puts message
+        else
+          puts "#{@stack_name.inspect} stack does not exist".color(:red)
+        end
+      end
+    end
+  end
+end

data/lib/pipedream/deploy.rb

@@ -0,0 +1,40 @@
+module Pipedream
+  class Deploy < Stack
+    def run
+      handle_rollback_completed!
+      if stack_exists?(@stack_name)
+        Update.new(@options).run
+      else
+        Create.new(@options).run
+      end
+    end
+
+    def handle_rollback_completed!
+      @stack = find_stack(@stack_name)
+      if @stack && rollback_complete?(@stack)
+        puts "Existing stack in ROLLBACK_COMPLETE state. Deleting stack before continuing."
+        cfn.delete_stack(stack_name: @stack_name)
+        status.wait
+        status.reset
+        @stack = nil # at this point stack has been deleted
+      end
+    end
+
+    def rollback_complete?(stack)
+      stack.stack_status == 'ROLLBACK_COMPLETE'
+    end
+
+    def find_stack(stack_name)
+      return if ENV['TEST']
+      resp = cfn.describe_stacks(stack_name: stack_name)
+      resp.stacks.first
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      # example: Stack with id demo-web does not exist
+      if e.message =~ /Stack with/ && e.message =~ /does not exist/
+        nil
+      else
+        raise
+      end
+    end
+  end
+end

data/lib/pipedream/dsl/pipeline/approve.rb

@@ -0,0 +1,34 @@
+module Pipedream::Dsl::Pipeline
+  module Approve
+    def approve(props)
+      default = {
+        name: "approve",
+        action_type_id: {
+          category: "Approval",
+          owner: "AWS",
+          provider: "Manual",
+          version: "1",
+        },
+        run_order: @run_order,
+        configuration: {  # required: will be set
+          notification_arn: {ref: "SnsTopic"}, # defaults to generated SNS topic
+        },
+      }
+
+      # Normalize special options. Simple approach of setting the default
+      case props
+      when String, Symbol
+        default[:configuration][:custom_data] = props
+        props = {}
+      when Hash
+        default[:configuration][:notification_arn] = props.delete(:notification_arn) if props.key?(:notification_arn)
+        default[:configuration][:custom_data] = props.delete(:custom_data) if props.key?(:custom_data)
+      else
+        raise "Invalid props type: #{props.class}"
+      end
+
+      options = default.merge(props)
+      action(options)
+    end
+  end
+end

data/lib/pipedream/dsl/pipeline/codebuild.rb

@@ -0,0 +1,57 @@
+module Pipedream::Dsl::Pipeline
+  module Codebuild
+    def codebuild(*projects)
+      default = {
+        # name: '', # will be set
+        action_type_id: {
+          category: "Build",
+          owner: "AWS",
+          provider: "CodeBuild",
+          version: "1",
+        },
+        run_order: @run_order,
+        # configuration: { project_name: '' }, # will be set
+        # output_artifacts: [name: "BuildArtifact#{name}"], # TODO: maybe make this configurable with a setting
+        input_artifacts: [name: "SourceArtifact"],
+      }
+
+      actions = projects.map do |item|
+        if item.is_a?(String)
+          name = item.underscore.camelize
+          project_name = adjusted_project_name(item) # add prefix and suffix
+          default.deep_merge(
+            name: name,
+            configuration: { project_name: project_name },
+          )
+        else # Hash
+          # With the hash, the user needs to set: name and configuration.project_name
+
+          # Handy shorthands
+          # The project name will allow this syntax
+          #   codebuild(name: "action-name", project_name: "codebuild-project-names")
+          project_name = item.delete(:project_name)
+          if project_name
+            item[:configuration] = { project_name: project_name }
+          end
+
+          item.reverse_merge(default)
+        end
+      end
+
+      action(*actions)
+    end
+
+    def codebuild_prefix(v)
+      @codebuild_prefix = v
+    end
+
+    def codebuild_suffix(v)
+      @codebuild_suffix = v
+    end
+
+  private
+    def adjusted_project_name(name)
+      [@codebuild_prefix, name, @codebuild_suffix].compact.join
+    end
+  end
+end

data/lib/pipedream/dsl/pipeline/github.rb

@@ -0,0 +1,42 @@
+module Pipedream::Dsl::Pipeline
+  module Github
+    def github(props)
+      # nice shorthands
+      source = props.delete(:source)
+      source = extract_repo_source(source)
+      owner,repo = source.split("/")
+
+      # cli option can override this in codepipe/pipeline.rb set_source!
+      # so cli option always gets the highest precendence
+      branch = props.delete(:branch) || "master" # always delete branch prop
+
+      o_auth_token = props.delete(:auth_token)
+      poll_for_source_changes = props.delete(:poll_for_source_changes) || "false"
+
+      default = {
+        name: "Source",
+        action_type_id: {
+          category: "Source",
+          owner: "ThirdParty",
+          provider: "GitHub",
+          version: "1",
+        },
+        run_order: @run_order,
+        configuration: {
+          branch: branch,
+          o_auth_token: o_auth_token,
+          owner: owner,
+          poll_for_source_changes: poll_for_source_changes,
+          repo: repo,
+        },
+        output_artifacts: [name: "SourceArtifact"]
+      }
+      action(props.reverse_merge(default))
+    end
+
+    def extract_repo_source(url)
+      url.sub('git@github.com:','').sub('https://github.com/','').sub(/\.git$/,'')
+    end
+    extend self # mainly for extract_repo_source
+  end
+end

data/lib/pipedream/dsl/pipeline.rb

@@ -0,0 +1,37 @@
+module Pipedream::Dsl
+  module Pipeline
+    include Approve
+    include Codebuild
+    include Github
+    include Ssm
+
+    PROPERTIES = %w[
+      artifact_store
+      artifact_stores
+      disable_inboundstage_transitions
+      name
+      restart_execution_on_update
+      role_arn
+      stages
+    ]
+    PROPERTIES.each do |prop|
+      define_method(prop) do |v|
+        @properties[prop.to_sym] = v
+      end
+    end
+
+    def stage(name, &block)
+      # Reset values for each stage declaraion
+      @run_order = 1
+
+      @current_stage = {name: name, actions: []}
+      @stages << @current_stage
+      block.call
+    end
+
+    def action(*props)
+      @current_stage[:actions] += props
+      @run_order += 1
+    end
+  end
+end

data/lib/pipedream/dsl/role.rb

@@ -0,0 +1,50 @@
+module Pipedream::Dsl
+  module Role
+    PROPERTIES = %w[
+      assume_role_policy_document
+      managed_policy_arns
+      max_session_duration
+      path
+      permissions_boundary
+      policies
+      role_name
+    ]
+    PROPERTIES.each do |prop|
+      define_method(prop) do |v|
+        @properties[prop.to_sym] = v
+      end
+    end
+
+    # convenience wrapper methods
+    def iam_policy(*definitions)
+      @iam_statements = definitions.map { |definition| standardize_iam_policy(definition) }
+    end
+
+    # Returns standarized IAM statement
+    def standardize_iam_policy(definition)
+      case definition
+      when String
+        # Expands simple string from: logs => logs:*
+        definition = "#{definition}:*" unless definition.include?(':')
+        {
+          action: [definition],
+          effect: "Allow",
+          resource: "*",
+        }
+      when Hash
+        definition
+      end
+    end
+
+    def managed_iam_policy(*definitions)
+      @managed_policy_arns = definitions.map { |definition| standardize_managed_iam_policy(definition) }
+    end
+
+    # AmazonEC2ReadOnlyAccess => arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess
+    def standardize_managed_iam_policy(definition)
+      return definition if definition.include?('iam::aws:policy')
+
+      "arn:aws:iam::aws:policy/#{definition}"
+    end
+  end
+end

data/lib/pipedream/dsl/schedule.rb

@@ -0,0 +1,30 @@
+module Pipedream::Dsl
+  module Schedule
+    PROPERTIES = %w[
+      description
+      event_pattern
+      name
+      role_arn
+      schedule_expression
+      state
+      targets
+    ]
+    PROPERTIES.each do |prop|
+      define_method(prop) do |v|
+        @properties[prop.to_sym] = v
+      end
+    end
+
+    def rate(period)
+      @schedule_expression = "rate(#{period})"
+    end
+
+    def cron(expression)
+      @schedule_expression = "cron(#{expression})"
+    end
+
+    def rule_event(props={})
+      @rule_event_props = props
+    end
+  end
+end

data/lib/pipedream/dsl/sns.rb

@@ -0,0 +1,15 @@
+module Pipedream::Dsl
+  module Sns
+    PROPERTIES = %w[
+      display_name
+      kms_master_key_id
+      subscription
+      topic_name
+    ]
+    PROPERTIES.each do |prop|
+      define_method(prop) do |v|
+        @properties[prop.to_sym] = v
+      end
+    end
+  end
+end

data/lib/pipedream/dsl/ssm.rb

@@ -0,0 +1,22 @@
+require "aws-sdk-ssm"
+
+module Pipedream::Dsl
+  module Ssm
+    # This method grabs the ssm parameter store value at "compile" time vs
+    # CloudFormation run time. In case we need it as part of the DSL compile phase.
+    def ssm(name)
+      resp = ssm_client.get_parameter(name: name)
+      if resp.parameter.type == "SecureString"
+        resp = ssm_client.get_parameter(name: name, with_decryption: true)
+      end
+
+      resp.parameter.value
+    rescue Aws::SSM::Errors::ParameterNotFound
+      puts "WARN: #{name} found on AWS SSM.".color(:yellow)
+    end
+
+    def ssm_client
+      @ssm_client ||= Aws::SSM::Client.new
+    end
+  end
+end