pipely 0.8.3 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 14e0eda2e643cd0b9b8fcf79b668adb5162510a8
-  data.tar.gz: ea9199edd89bcbc3d8b6ab71a727e2a8c69e2b56
+  metadata.gz: 6ed7918ddfbf5b2cd544a8e4a6513d0922811456
+  data.tar.gz: dde98acf09526facedfa74750fb5ad9bdfbac81a
 SHA512:
-  metadata.gz: f7f3a6df462c844a132d070af1474c3b102c40ad4592531f8e9078f5523abf0b265386b8c090a12c2dd39f18d8060d9f2d1f47b448a5eb8080dadc97fc3c9b2d
-  data.tar.gz: d578ec42ba4eb48e6937824b2a67c6af16a7b3c700655a3d956e9584f7cce3eb1ea33b1fb8109f8a26c2db36abaa544375a67b0fbad6515d3621e43a7e508078
+  metadata.gz: 5b4423e586a2079f7ee3c65c4463fb577e109f8dc66e84be105ce3f45237f2e5afab85144399a83a3768f401bf16b72a01a5a3e399e63b181d4fca2ded74e2ba
+  data.tar.gz: 1fda022cab04e00666bf7ed4776f6b7bc35e0e6945f38d276c8cfc693dc654a7c27d738954b016534c69cfa15d265bd0c3b88c50c82c7c3ba055adc2440a8712

data/lib/pipely/build.rb

@@ -4,6 +4,7 @@ require 'pipely/build/daily_scheduler'
 require 'pipely/build/right_now_scheduler'
 require 'pipely/build/s3_path_builder'
 require 'pipely/build/environment_config'
+require 'pathology'
 
 module Pipely
 
@@ -15,22 +16,7 @@ module Pipely
       env = environment.to_sym
       config = EnvironmentConfig.load(config_path, env)
 
-      case environment.to_sym
-      when :production
-        s3_prefix = "production/#{config[:namespace]}"
-        if config[:start_time]
-          # allow config to change pipelint start time
-          # TODO: all scheduling should be done through config before pipely 1.0
-          scheduler = DailyScheduler.new(config[:start_time])
-        else
-          scheduler = DailyScheduler.new
-        end
-      when :staging
-        s3_prefix = "staging/#{`whoami`.strip}/#{config[:namespace]}"
-        scheduler = RightNowScheduler.new
-      end
-
-      Definition.new(template, env, s3_prefix, scheduler, config)
+      Definition.new(template, env, config)
     end
 
   end

data/lib/pipely/build/daily_scheduler.rb

@@ -6,7 +6,7 @@ module Pipely
     #
     class DailyScheduler
 
-      def initialize(start_time="11:00:00")
+      def initialize(start_time)
         @start_time = start_time
       end
 

data/lib/pipely/build/definition.rb

@@ -3,7 +3,7 @@ module Pipely
 
     # Represent a pipeline definition, built from a Template and some config.
     #
-    class Definition < Struct.new(:template,:env,:s3_prefix,:scheduler,:config)
+    class Definition < Struct.new(:template, :env, :config)
       def pipeline_name
         config[:name]
       end
@@ -12,6 +12,15 @@ module Pipely
         config[:namespace]
       end
 
+      def s3_prefix
+        if config[:s3_prefix]
+          template = Pathology.template(config[:s3_prefix])
+          template.interpolate(interpolation_context)
+        else
+          fail('unspecified s3_prefix')
+        end
+      end
+
       def s3_path_builder
         S3PathBuilder.new(config[:s3].merge(prefix: s3_prefix))
       end
@@ -24,7 +33,26 @@ module Pipely
 
         template.to_json
       end
-    end
 
+      def scheduler
+        case config[:scheduler]
+        when 'daily'
+          DailyScheduler.new(config[:start_time])
+        when 'now'
+          RightNowScheduler.new
+        else
+          fail('unspecified scheduler')
+        end
+      end
+
+    private
+
+      def interpolation_context
+        config.merge({
+          :whoami => `whoami`.strip,
+        })
+      end
+
+    end
   end
 end

data/lib/pipely/build/environment_config.rb

@@ -8,9 +8,32 @@ module Pipely
     #
     class EnvironmentConfig < Hash
 
+      # Continue supporting env-based defaults until pipely v1.0
+      ENV_DEFAULTS = {
+        production: {
+          s3_prefix: 'production/:namespace',
+          scheduler: 'daily',
+          start_time: '11:00:00',
+        },
+        staging: {
+          s3_prefix: 'staging/:whoami/:namespace',
+          scheduler: 'now',
+
+          # Since scheduler can now be overridden via commandline argument,
+          # supply a start_time even for environments that default to 'now'.
+          start_time: '11:00:00',
+        }
+      }
+
       def self.load(filename, environment)
         raw = YAML.load_file(filename)[environment.to_s]
-        load_from_hash(raw)
+        config = load_from_hash(raw)
+
+        if defaults = ENV_DEFAULTS[environment.to_sym]
+          defaults.merge(config)
+        else
+          config
+        end
       end
 
       def self.load_from_hash(attributes)

data/lib/pipely/build/s3_path_builder.rb

@@ -1,3 +1,5 @@
+require 'pathology'
+
 module Pipely
   module Build
 
@@ -5,56 +7,86 @@ module Pipely
     #
     class S3PathBuilder
 
-      attr_reader :assets_bucket, :logs_bucket, :steps_bucket
-
       START_TIME = "\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}"
       START_DATE = "\#{format(@scheduledStartTime,'YYYY-MM-dd')}"
 
+      # options[:templates] should contain a Hash of your desired S3 path
+      # patterns, formatted for Pathology.  The remainder of the options Hash
+      # serves as interpolation values for the templates.
+      #
+      # Several additional interpolation variables (:protocol, :timestamp,
+      # :datestamp) are provided by S3PathBuilder at interpolation time.
+      #
+      # If options[:templates] is not present, or if it is missing any of the
+      # legacy templates (assets, logs, steps, etc.), they will be
+      # automatically built, using bucket names found in the options Hash,
+      # preserving the original behavior.
+      #
       def initialize(options)
-        @assets_bucket = options[:assets]
-        @logs_bucket = options[:logs]
-        @steps_bucket = options[:steps]
-        @s3prefix = options[:prefix]
-      end
+        @options = options.merge({
+          timestamp: START_TIME,
+          datestamp: START_DATE,
+        })
 
-      def s3_log_prefix
-        "s3://#{@logs_bucket}/#{@s3prefix}/#{START_TIME}"
-      end
-
-      def s3_step_prefix
-        "s3://#{@steps_bucket}/#{@s3prefix}"
-      end
+        @path_templates = default_templates
 
-      def s3n_step_prefix
-        "s3n://#{@steps_bucket}/#{@s3prefix}"
+        if templates = @options.delete(:templates)
+          @path_templates.merge!(templates)
+        end
       end
 
-      def s3_asset_prefix
-        "s3://#{@assets_bucket}/#{bucket_relative_s3_asset_prefix}"
-      end
+      # Support legacy interface, wherein config simply contained bucket names,
+      # and users were forced to abide by Pipely's somewhat arbitrary path
+      # structure.
+      #
+      def default_templates
+        assets, logs, steps = @options.values_at(:assets, :logs, :steps)
 
-      def s3n_asset_prefix
-        "s3n://#{@assets_bucket}/#{@s3prefix}/#{START_TIME}"
+        {
+          asset: ":protocol://#{assets}/:prefix/:timestamp",
+          log: ":protocol://#{logs}/:prefix/:timestamp",
+          step: ":protocol://#{steps}/:prefix",
+          shared_asset: ":protocol://#{assets}/:prefix/shared/:datestamp",
+          bucket_relative_asset: ':prefix/:timestamp',
+        }
       end
 
-      def s3_shared_asset_prefix
-        "s3://#{@assets_bucket}/#{@s3prefix}/shared/#{START_DATE}"
+      # Implement path interpolation methods, e.g. s3_log_prefix, etc.
+      #
+      def method_missing(method_name, *args, &block)
+        case method_name
+        when /^(s3n?)_(.*)_prefix$/
+          if pattern = @path_templates[$2.to_sym]
+            Pathology.template(pattern).interpolate(
+              @options.merge({protocol: $1})
+            )
+          else
+            super
+          end
+        else
+          super
+        end
       end
 
+      # Re-route legacy method name to the standard format implemented by
+      # method_missing above.
+      #
       def bucket_relative_s3_asset_prefix
-        "#{@s3prefix}/#{START_TIME}"
+        s3_bucket_relative_asset_prefix
       end
 
       def to_hash
-        {
-          :s3_log_prefix => s3_log_prefix,
-          :s3_step_prefix => s3_step_prefix,
-          :s3n_step_prefix => s3n_step_prefix,
-          :s3_asset_prefix => s3_asset_prefix,
-          :s3n_asset_prefix => s3n_asset_prefix,
-          :s3_shared_asset_prefix => s3_shared_asset_prefix,
-          :bucket_relative_s3_asset_prefix => bucket_relative_s3_asset_prefix,
-        }
+        values = %w(s3 s3n).flat_map do |protocol|
+          @path_templates.keys.map do |path_name|
+            key = "#{protocol}_#{path_name}_prefix".to_sym
+            [key, send(key)]
+          end
+        end
+
+        # Support legacy method name.
+        Hash[values].merge({
+          bucket_relative_s3_asset_prefix: bucket_relative_s3_asset_prefix
+        })
       end
 
     end

data/lib/pipely/deploy/bootstrap.rb

@@ -1,6 +1,8 @@
 require 'pipely/bundler'
 require 'pipely/deploy/bootstrap_context'
+require 'pipely/deploy/bootstrap_registry'
 require 'pipely/deploy/s3_uploader'
+require 'active_support/core_ext/string/conversions'
 
 module Pipely
   module Deploy
@@ -8,23 +10,24 @@ module Pipely
     # Helps bootstrap a pipeline
     class Bootstrap
 
-      attr_reader :project_spec
-      attr_reader :gem_files
+      attr_reader :gem_files, :s3_steps_path
 
-      def initialize(s3_uploader)
-        @s3_uploader = s3_uploader
+      def initialize(gem_files, s3_steps_path)
+        @gem_files = gem_files
+        @s3_steps_path = s3_steps_path
       end
 
-      # Builds the project's gem from gemspec, uploads the gem to s3, and
-      # uploads all the gem dependences to S3
-      def build_and_upload_gems
-        @gem_files = Pipely::Bundler.gem_files
-        @s3_uploader.upload(@gem_files.values)
-      end
-
-      def context(s3_steps_path)
-        BootstrapContext.new.tap do |context|
-          context.gem_files = @s3_uploader.s3_urls(gem_files.values)
+      def context(*mixins)
+        bootstrap_mixins = BootstrapRegistry.instance.register_mixins(mixins)
+
+        BootstrapContext.class_eval do
+          bootstrap_mixins.each do |mixin|
+            puts "Adding bootstrap mixin #{mixin}"
+            include mixin.constantize
+          end
+          self
+        end.new.tap do |context|
+          context.gem_files = gem_files
           context.s3_steps_path = s3_steps_path
         end
       end

data/lib/pipely/deploy/bootstrap_context.rb

@@ -2,27 +2,104 @@
 module Pipely
   module Deploy
 
-    # Context passed to the erb templates
+    # Context passed to the erb templates, providers helpers for
+    # common bootstraping activities for emr and ec2 instances.
+    #
+    #   bootstrap.ec2.install_gems_script
+    #   bootstrap.emr.install_gems_script
+    #
     class BootstrapContext
-      attr_accessor :gem_files
-      attr_accessor :s3_steps_path
+      attr_accessor :gem_files, :s3_steps_path
+      attr_reader :ec2, :emr
 
-      def install_gems_script(transport = :hadoop_fs, &blk)
-        script = ""
+      # Context for EMR instances
+      class EmrContext
+        def initialize(parent)
+          @parent = parent
+        end
+
+        def install_gems_script(&blk)
+          @parent.install_gems_script(:hadoop_fs, &blk)
+        end
+      end
+
+      # Context for EC2 instances
+      class Ec2Context
+        def initialize(parent)
+          @parent = parent
+          @ssh_initialized = false
+        end
+
+        def install_gems_script(&blk)
+          @parent.install_gems_script(:awscli, &blk)
+        end
+
+        def as_root(init_ssh=true)
+          script = ""
+
+          if init_ssh && !@ssh_initialized
+            @ssh_initialized = true
+            script << %{
+# Set up ssh access
+if [ ! -f ~/.ssh/id_rsa ]; then
+  mkdir -p ~/.ssh
+  ssh-keygen -P '' -f ~/.ssh/id_rsa
+  cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
+  chmod 600 ~/.ssh/authorized_keys
+fi
+}
+          end
+
+          script << %{
+# Use ssh to bypass the sudo "require tty" setting
+ssh -o "StrictHostKeyChecking no" -t -t ec2-user@localhost <<- EOF
+  sudo su -;
+}
+
+          # The yield to be run as root
+          script << yield
 
+          script << %{
+  # exit twice, once for su and once for ssh
+  exit;
+  exit;
+EOF
+}
+        end
+      end
+
+      def initialize
+        @emr = EmrContext.new(self)
+        @ec2 = Ec2Context.new(self)
+      end
+
+      def fetch_command(transport)
         case transport.to_sym
         when :hadoop_fs
-          transport_cmd = 'hadoop fs -copyToLocal'
+          'hadoop fs -copyToLocal'
         when :awscli
-          transport_cmd = 'aws s3 cp'
-        else
+          'aws s3 cp'
+        end
+      end
+
+      def install_gems_script(transport, &blk)
+
+        transport_cmd = fetch_command(transport)
+
+        if transport_cmd.nil?
           raise "Unsupported transport: #{transport}" unless blk
         end
 
+        script = ""
         @gem_files.each do |gem_file|
           filename = File.basename(gem_file)
-          command = "#{transport_cmd} #{gem_file} #{filename}" if transport_cmd
-          command = yield(gem_file, filename, command) if blk
+          params = [transport_cmd, gem_file, filename]
+          if blk
+            command = yield(*params)
+          else
+            command = params.join(" ")
+          end
+
           script << %Q[
 # #{filename}
 #{command}

data/lib/pipely/deploy/bootstrap_registry.rb

@@ -0,0 +1,45 @@
+require 'singleton'
+require 'active_support/core_ext/string/conversions'
+
+module Pipely
+  module Deploy
+
+    #
+    ## Registry of Mixins to be applied to the bootstrap context
+    #
+    class BootstrapRegistry
+      include Singleton
+
+      def initialize
+        @mixins = []
+      end
+
+      class << self
+        def register_mixins(*mixins)
+          instance.register_mixins(*mixins)
+        end
+
+        def mixins
+          instance.mixins
+        end
+      end
+
+      def register_mixins(*mixins)
+        new_mixins = [mixins].flatten.compact
+
+        new_mixins.each do |mixin|
+          begin
+            require mixin.underscore
+          rescue LoadError => e
+            raise "Failed to require #{mixin} for bootstrap_contexts: #{e}"
+          end
+        end
+        @mixins = (@mixins + new_mixins).uniq
+      end
+
+      def mixins
+        @mixins
+      end
+    end
+  end
+end