pipely 0.8.3 → 0.10.0

data/lib/pipely/deploy/client.rb

@@ -15,6 +15,7 @@ require 'aws-sdk'
 require 'logger'
 require 'tempfile'
 require 'securerandom'
+require 'pipely/deploy/json_definition'
 
 module Pipely
   module Deploy
@@ -49,16 +50,18 @@ module Pipely
         created_pipeline_id = create_pipeline(pipeline_name,
                                               definition,
                                               tags)
-        @log.info("Created pipeline id '#{created_pipeline_id}'")
-
-        # Delete old pipelines
-        pipeline_ids.each do |pipeline_id|
-          begin
-            delete_pipeline(pipeline_id)
-            @log.info("Deleted pipeline '#{pipeline_id}'")
-
-          rescue PipelineDeployerError => error
-            @log.warn(error)
+        if created_pipeline_id
+          @log.info("Created pipeline id '#{created_pipeline_id}'")
+
+          # Delete old pipelines
+          pipeline_ids.each do |pipeline_id|
+            begin
+              delete_pipeline(pipeline_id)
+              @log.info("Deleted pipeline '#{pipeline_id}'")
+
+            rescue PipelineDeployerError => error
+              @log.warn(error)
+            end
           end
         end
 
@@ -83,20 +86,32 @@ module Pipely
       end
 
       def create_pipeline(pipeline_name, definition, tags={})
-        definition_objects = JSON.parse(definition)['objects']
-
-        unique_id = SecureRandom.uuid
-
+        # Use Fog gem, instead of aws-sdk gem, to create pipeline with tags.
+        #
+        # TODO: Consolidate on aws-sdk when tagging support is added.
+        #
         created_pipeline = @data_pipelines.pipelines.create(
-          unique_id: unique_id,
+          unique_id: SecureRandom.uuid,
           name: pipeline_name,
           tags: default_tags.merge(tags)
         )
 
-        created_pipeline.put(definition_objects)
-        created_pipeline.activate
+        # Use aws-sdk gem, instead of Fog, to put definition and activate
+        # pipeline, for improved reporting of validation errors.
+        #
+        response = @aws.put_pipeline_definition(
+          pipeline_id: created_pipeline.id,
+          pipeline_objects: JSONDefinition.parse(definition)
+        )
 
-        created_pipeline.id
+        if response[:errored]
+          @log.error("Failed to put pipeline definition.")
+          @log.error(response[:validation_errors].inspect)
+          false
+        else
+          @aws.activate_pipeline(pipeline_id: created_pipeline.id)
+          created_pipeline.id
+        end
       end
 
       def delete_pipeline(pipeline_id)

data/lib/pipely/deploy/json_definition.rb

@@ -0,0 +1,51 @@
+require 'json'
+
+module Pipely
+  module Deploy
+
+    # The JSON definition format expected by the CLI differs from the structure
+    # expected by the API.  This class transforms a CLI-ready definition into
+    # the pipeline object hashes expected by the API.
+    #
+    class JSONDefinition
+      def self.parse(definition)
+        definition_objects = JSON.parse(definition)['objects']
+        definition_objects.map { |object| new(object).to_api }
+      end
+
+      def initialize(object)
+        @json_fields = object.clone
+        @id = @json_fields.delete('id')
+        @name = @json_fields.delete('name') || @id
+      end
+
+      def to_api
+        {
+          'id' => @id,
+          'name' => @name,
+          'fields' => fields
+        }
+      end
+
+      private
+
+      def fields
+        @json_fields.map{|k,v| field_for_kv(k,v)}.flatten
+      end
+
+      def field_for_kv(key, value)
+        if value.is_a?(Hash)
+          { 'key' => key, 'refValue' => value['ref'] }
+
+        elsif value.is_a?(Array)
+          value.map { |subvalue| field_for_kv(key, subvalue) }
+
+        else
+          { 'key' => key, 'stringValue' => value }
+
+        end
+      end
+    end
+
+  end
+end

data/lib/pipely/pipeline_date_time/pipeline_date.rb

@@ -0,0 +1,62 @@
+# encoding: utf-8
+module Pipely
+  module PipelineDateTime
+    # Encapsulates AWS pipeline date
+    #
+    class PipelineDate
+      DEFAULT_DAY_FORMAT = 'YYYY/MM/dd'
+      DEFAULT_MONTH_FORMAT = 'YYYY/MM'
+      DEFAULT_YEAR_FORMAT = 'YYYY'
+
+      class << self
+        def day_format=(day_format)
+          @day_format = day_format
+        end
+
+        def day_format
+          @day_format || DEFAULT_DAY_FORMAT
+        end
+
+        def month_format=(month_format)
+          @month_format = month_format
+        end
+
+        def month_format
+          @month_format || DEFAULT_MONTH_FORMAT
+        end
+
+        def year_format=(year_format)
+          @year_format = year_format
+        end
+
+        def year_format
+          @year_format || DEFAULT_YEAR_FORMAT
+        end
+      end
+
+      def initialize(target_date, days_back)
+        days_back = days_back.to_i
+        @date_expression = case
+                           when days_back > 0
+                             "minusDays(#{target_date}, #{days_back})"
+                           when days_back == 0
+                             target_date
+                           else
+                             "plusDays(#{target_date}, #{-days_back})"
+                           end
+      end
+
+      def day
+        "\#{format(#{@date_expression}, \"#{PipelineDate.day_format}\")}"
+      end
+
+      def month
+        "\#{format(#{@date_expression}, \"#{PipelineDate.month_format}\")}"
+      end
+
+      def year
+        "\#{format(#{@date_expression}, \"#{PipelineDate.year_format}\")}"
+      end
+    end
+  end
+end

data/lib/pipely/pipeline_date_time/pipeline_date_pattern.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+require 'pipely/pipeline_date_time/pipeline_day_range'
+require 'pipely/pipeline_date_time/pipeline_month_range'
+require 'pipely/pipeline_date_time/pipeline_year_range'
+
+module Pipely
+  module PipelineDateTime
+    # Mixin for constructing compact date pattern selections
+    #
+    module PipelineDatePattern
+      def date_pattern
+        selection.target_all_time ? '.*' : any_string(date_pattern_parts)
+      end
+
+      private
+
+      def date_pattern_parts
+        day_range.exclude(month_range.start, month_range.end)
+        month_range.exclude(year_range.start, year_range.end)
+        day_range.days + month_range.months + year_range.years
+      end
+
+      def day_range
+        @day_range ||= PipelineDayRange.new(selection.target_date, num_days, 0)
+      end
+
+      def month_range
+        @month_range ||= PipelineMonthRange.new(selection.target_date, num_days,
+                                                0)
+      end
+
+      def year_range
+        @year_range ||= PipelineYearRange.new(selection.target_date, num_days,
+                                              0)
+      end
+
+      def num_days
+        selection.num_days_back.to_i
+      end
+    end
+  end
+end

data/lib/pipely/pipeline_date_time/pipeline_date_range_base.rb

@@ -0,0 +1,44 @@
+# encoding: utf-8
+require 'pipely/pipeline_date_time/pipeline_date'
+
+module Pipely
+  module PipelineDateTime
+    # Base class for pipeline date ranges
+    #
+    class PipelineDateRangeBase
+      attr_reader :days_back
+
+      def initialize(target_date, days_back_start, days_back_end)
+        @target_date = target_date
+        @days_back_start = days_back_start
+        @days_back_end = days_back_end
+        @days_back = (days_back_end..days_back_start).to_set
+      end
+
+      def start
+        @days_back_start
+      end
+
+      def end
+        @days_back_end
+      end
+
+      def exclude(days_back_start, days_back_end)
+        return if days_back_start < 0
+        return if days_back_end < 0
+        return if days_back_start < days_back_end  # Back smaller for earlier
+        (days_back_end..days_back_start).each do |days_back|
+          @days_back.delete days_back
+        end
+      end
+
+      private
+
+      def pipeline_dates
+        @pipeline_dates ||= @days_back.map do |days_back|
+          PipelineDate.new(@target_date, days_back)
+        end
+      end
+    end
+  end
+end

data/lib/pipely/pipeline_date_time/pipeline_day_range.rb

@@ -0,0 +1,14 @@
+# encoding: utf-8
+require 'pipely/pipeline_date_time/pipeline_date_range_base'
+
+module Pipely
+  module PipelineDateTime
+    # Class that represents a range of individual pipeline days
+    #
+    class PipelineDayRange < PipelineDateRangeBase
+      def days
+        @days ||= pipeline_dates.map { |pd| pd.day }
+      end
+    end
+  end
+end

data/lib/pipely/pipeline_date_time/pipeline_month_range.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+require 'pipely/pipeline_date_time/pipeline_date_range_base'
+
+module Pipely
+  module PipelineDateTime
+    # Class that represents a range of individual pipeline months
+    #
+    class PipelineMonthRange < PipelineDateRangeBase
+      MINIMUM_MONTH_OFFSET = 30  # The month of x+/-30 will never add extra days
+      MONTH_INTERVAL = 28  # We never miss a month by taking every 28 days
+
+      attr_reader :start, :end
+
+      def initialize(target_date, days_back_start, days_back_end)
+        @target_date = target_date
+        @start = days_back_start - MINIMUM_MONTH_OFFSET
+        @end = days_back_end + MINIMUM_MONTH_OFFSET
+        @days_back = (@end..@start).step(MONTH_INTERVAL).to_set
+      end
+
+      def months
+        @months ||= pipeline_dates.map { |pd| pd.month }
+      end
+    end
+  end
+end

data/lib/pipely/pipeline_date_time/pipeline_year_range.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+require 'pipely/pipeline_date_time/pipeline_date_range_base'
+
+module Pipely
+  module PipelineDateTime
+    # Class that represents a range of individual pipeline years
+    #
+    class PipelineYearRange < PipelineDateRangeBase
+      DAYS_IN_YEAR = 365
+
+      attr_reader :start, :end
+
+      def initialize(target_date, days_back_start, days_back_end)
+        @target_date = target_date
+        @start = days_back_start - DAYS_IN_YEAR
+        @end = days_back_end + DAYS_IN_YEAR
+        @days_back = (@end..@start).step(DAYS_IN_YEAR).to_set
+      end
+
+      def years
+        @years ||= pipeline_dates.map { |pd| pd.year }
+      end
+    end
+  end
+end

data/lib/pipely/tasks/definition.rb

@@ -34,6 +34,9 @@ module Pipely
       def initialize(*args, &task_block)
         setup_ivars(args)
 
+        # First non-name parameter allows overriding the configured scheduler.
+        args.unshift(:scheduler)
+
         directory path
 
         desc "Generates the pipeline definition file"
@@ -43,6 +46,10 @@ module Pipely
               task_block.call(*[self, task_args].slice(0, task_block.arity))
             end
 
+            if scheduler_override = task_args[:scheduler]
+              definition.config[:scheduler] = scheduler_override
+            end
+
             run_task verbose
           end
         end

data/lib/pipely/tasks/deploy.rb

@@ -26,6 +26,9 @@ module Pipely
       def initialize(*args, &task_block)
         setup_ivars(args)
 
+        # First non-name parameter allows overriding the configured scheduler.
+        args.unshift(:scheduler)
+
         desc "Deploy pipeline" unless ::Rake.application.last_comment
 
         task name, *args do |_, task_args|
@@ -34,6 +37,10 @@ module Pipely
               task_block.call(*[self, task_args].slice(0, task_block.arity))
             end
 
+            if scheduler_override = task_args[:scheduler]
+              definition.config[:scheduler] = scheduler_override
+            end
+
             run_task verbose
           end
         end

data/lib/pipely/tasks/upload_pipeline_as_gem.rb

@@ -1,4 +1,5 @@
 require 'rake'
+require 'rake/tasklib'
 require 'aws'
 require 'erubis'
 require 'pipely/deploy/bootstrap'
@@ -15,6 +16,7 @@ module Pipely
       attr_accessor :s3_steps_path
       attr_accessor :s3_gems_path
       attr_accessor :config
+      attr_accessor :templates
 
       def initialize(*args, &task_block)
         setup_ivars(args)
@@ -35,12 +37,14 @@ module Pipely
       def setup_ivars(args)
         @name = args.shift || 'deploy:upload_pipeline_as_gem'
         @verbose = true
+        @templates = Dir.glob("templates/*.erb")
       end
 
       def run_task(verbose)
-        context = build_bootstrap_context
+        s3_gem_paths = upload_gems
+        context = build_bootstrap_context(s3_gem_paths)
 
-        Dir.glob("templates/*.erb").each do |erb_file|
+        templates.each do |erb_file|
           upload_filename = File.basename(erb_file).sub( /\.erb$/, '' )
 
           # Exclude the pipeline.json
@@ -55,18 +59,24 @@ module Pipely
 
       private
       def s3_bucket
-          s3 = AWS::S3.new
-          s3.buckets[@bucket_name]
+          @s3_bucket ||= AWS::S3.new.buckets[@bucket_name]
       end
 
-      def build_bootstrap_context
-        s3_uploader = Pipely::Deploy::S3Uploader.new(s3_bucket, @s3_gems_path)
-        bootstrap_helper = Pipely::Deploy::Bootstrap.new(s3_uploader)
-        bootstrap_helper.build_and_upload_gems
+      def upload_gems
+        pipeline_gems = Pipely::Bundler.gem_files
+        s3_uploader = Pipely::Deploy::S3Uploader.new(s3_bucket, s3_gems_path)
+        s3_uploader.upload(pipeline_gems.values)
+        s3_uploader.s3_urls(pipeline_gems.values)
+      end
+
+      def build_bootstrap_context(s3_gems)
+        bootstrap_helper = Pipely::Deploy::Bootstrap.new(s3_gems, s3_steps_path)
+
+        context = bootstrap_helper.context(config['bootstrap_mixins'])
 
         # erb context
         {
-          bootstrap: bootstrap_helper.context(@s3_steps_path),
+          bootstrap: context,
           config: config
         }
       end