pipely 0.13.0 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b731960bf5c0239cac639066f033a49a4e5e6c0f
-  data.tar.gz: 1612176e9731abbe4d00ad0e9d900a92fb0a3f37
+  metadata.gz: c7b441e79e14a90c03f6db511d4e8504ac5f8360
+  data.tar.gz: 692e36969e5b3100caa338d474127498ee870df6
 SHA512:
-  metadata.gz: 2fd1252c31ad76533f47cd2e5262e8e99710354af17308e50490773ca4670720dfeecab393938936db58a9f0a1b79656e88a8b143a9cd48b3223accd23958c1f
-  data.tar.gz: a4426e893afb1ea016504ef4853a452c99332b13b347810d66b2a9b0816fffe18eb60da85a6d83b7a1e771066bf3245dd8ed42d84e0b7d6e0d7b5a5b2d14b554
+  metadata.gz: 153c54d53fcbdc2f123616d5284e303df8f4b5b557520eb65d774d7873bbd206a90fbfb9e5e68235f37747eef030979b8bdd3dcaf745774702bfae6ea997b75c
+  data.tar.gz: 3f3b17eb81bc8e5b712cc5ddc5e23785e52c3cb7ba9f60882aec6069851e4ee4c97ff80c5b53c6b739c790c71228f788ddf7c025afeb214b417fd184f6f5226f

data/lib/pipely.rb

@@ -11,6 +11,8 @@ require 'pipely/s3_writer'
 #
 module Pipely
 
+  ENV['AWS_REGION'] ||= 'us-east-1'
+
   def self.draw(definition_json, filename, component_attributes=nil)
     definition = Definition.parse(definition_json)
 

data/lib/pipely/actions/list_live_pipelines.rb

@@ -14,31 +14,30 @@ module Pipely
           $stdout.puts pipeline_ids.to_json
         else
           $stdout.puts pipeline_ids.map { |pipeline|
-            [ pipeline['name'], pipeline['id'] ].join("\t")
+            [ pipeline.name, pipeline.id ].join("\t")
           }
         end
       end
 
-    private
+      private
 
       def pipeline_ids
         ids = []
 
+        data_pipeline = Aws::DataPipeline::Client.new
+
+
+        marker = nil
         begin
-          result = data_pipeline.list_pipelines
-          ids += result['pipelineIdList']
-        end while (result['hasMoreResults'] && result['marker'])
+          result = data_pipeline.list_pipelines(
+            marker: marker,
+          )
+          ids += result.pipeline_id_list
+          marker = result.marker
+        end while (result.has_more_results && marker)
 
         ids
       end
-
-      def data_pipeline
-        Fog::AWS::DataPipeline.new
-      rescue ArgumentError
-        $stderr.puts "#{self.class.name}: Falling back to IAM profile"
-        Fog::AWS::DataPipeline.new(use_iam_profile: true)
-      end
-
     end
 
   end

data/lib/pipely/deploy/client.rb

@@ -1,16 +1,3 @@
-# Note: We are in the process of migrating from Fog to aws-sdk for communicating
-# with the Data Pipeline API.  aws-sdk offers several benefits, such as:
-#
-# * Built-in automated exponential back-off, to avoid hitting rate limits.
-# * Working pagination of ListPipelines responses.
-# * Authentication using IAM resource roles.
-# * Faster installation.
-#
-# On the downside, aws-sdk does not yet support tagging of pipelines.  We can
-# not yet port pipely entirely away from Fog until this is resolved, so we will
-# temporarily require both libraries.
-
-require 'fog'
 require 'aws-sdk'
 require 'logger'
 require 'tempfile'
@@ -24,23 +11,26 @@ module Pipely
     #
     class Client
 
+      attr_reader :base_tags
+
       # Generic error representing failure to deploy a rendered definition.
       class PipelineDeployerError < RuntimeError; end
 
       def initialize(log=nil)
         @log = log || Logger.new(STDOUT)
-        @data_pipelines = Fog::AWS::DataPipeline.new
-        @aws = AWS::DataPipeline.new.client
+        @aws = Aws::DataPipeline::Client.new
+        @base_tags = {
+          "environment" => ENV['env'],
+          "creator" => ENV['USER']
+        }
       end
 
-      def deploy_pipeline(pipeline_basename, definition=nil, &block)
-        pipeline_name = [
-          ('P' if ENV['env'] == 'production'),
-          ENV['USER'],
-          pipeline_basename
-        ].compact.join(':')
+      def deploy_pipeline(pipeline_basename, definition = nil, &block)
+        pipeline_name = pipeline_name(pipeline_basename)
 
-        tags = { "basename" => pipeline_basename }
+        tags = base_tags.merge(
+          "basename" => pipeline_basename,
+          "deploy_id" => SecureRandom.uuid )
 
         # Get a list of all existing pipelines
         pipeline_ids = existing_pipelines(pipeline_name)
@@ -87,23 +77,19 @@ module Pipely
       end
 
       def create_pipeline(pipeline_name, definition, tags={})
-        # Use Fog gem, instead of aws-sdk gem, to create pipeline with tags.
-        #
-        # TODO: Consolidate on aws-sdk when tagging support is added.
-        #
-        created_pipeline = @data_pipelines.pipelines.create(
-          unique_id: SecureRandom.uuid,
+        created_pipeline = @aws.create_pipeline(
           name: pipeline_name,
-          tags: default_tags.merge(tags)
+          unique_id: tags['deploy_id'] || SecureRandom.uuid,
+          description: "Pipely Deployed Data Pipeline",
+          tags: base_tags.merge(tags).map do |k,v|
+            { key: k, value: v } unless v.nil?
+          end.compact,
         )
 
-        definition ||= yield(created_pipeline.id) if block_given?
+        definition ||= yield(created_pipeline.pipeline_id) if block_given?
 
-        # Use aws-sdk gem, instead of Fog, to put definition and activate
-        # pipeline, for improved reporting of validation errors.
-        #
         response = @aws.put_pipeline_definition(
-          pipeline_id: created_pipeline.id,
+          pipeline_id: created_pipeline.pipeline_id,
           pipeline_objects: JSONDefinition.parse(definition)
         )
 
@@ -116,25 +102,24 @@ module Pipely
           @log.error(response[:validation_errors].inspect)
           false
         else
-          @aws.activate_pipeline(pipeline_id: pipeline.id)
-          pipeline.id
+          @aws.activate_pipeline(pipeline_id: pipeline.pipeline_id)
+          pipeline.pipeline_id
         end
       end
 
       def delete_pipeline(pipeline_id)
-        @data_pipelines.pipelines.get(pipeline_id).destroy
+        @aws.delete_pipeline(pipeline_id: pipeline_id)
       end
 
-    private
+      private
 
-      def default_tags
-        {
-          "environment" => ENV['env'],
-          "creator" => ENV['USER'],
-          "deploy_id" => SecureRandom.uuid
-        }
+      def pipeline_name(basename)
+        [
+          ('P' if ENV['env'] == 'production'),
+          ENV['USER'],
+          basename
+        ].compact.join(':')
       end
-
     end
   end
 end

data/lib/pipely/deploy/json_definition.rb

@@ -9,21 +9,22 @@ module Pipely
     #
     class JSONDefinition
       def self.parse(definition)
-        definition_objects = JSON.parse(definition)['objects']
+        definition_objects =
+          JSON.parse(definition, symbolize_names: true)[:objects]
         definition_objects.map { |object| new(object).to_api }
       end
 
       def initialize(object)
         @json_fields = object.clone
-        @id = @json_fields.delete('id')
-        @name = @json_fields.delete('name') || @id
+        @id = @json_fields.delete(:id)
+        @name = @json_fields.delete(:name) || @id
       end
 
       def to_api
         {
-          'id' => @id,
-          'name' => @name,
-          'fields' => fields
+          id: @id,
+          name: @name,
+          fields: fields
         }
       end
 
@@ -35,13 +36,13 @@ module Pipely
 
       def field_for_kv(key, value)
         if value.is_a?(Hash)
-          { 'key' => key, 'refValue' => value['ref'] }
+          { key: key, ref_value: value[:ref] }
 
         elsif value.is_a?(Array)
           value.map { |subvalue| field_for_kv(key, subvalue) }
 
         else
-          { 'key' => key, 'stringValue' => value }
+          { key: key, string_value: value }
 
         end
       end

data/lib/pipely/deploy/s3_uploader.rb

@@ -39,7 +39,7 @@ module Pipely
       #
       def upload_file(file)
         target_path = s3_file_path(file)
-        s3_object = @s3_bucket.objects[target_path]
+        s3_object = @s3_bucket.object(target_path)
 
         content = File.read(file)
         digest = Digest::MD5.hexdigest(content)
@@ -48,7 +48,7 @@ module Pipely
           puts "skipping #{file} to #{target_path} (ETAG matches)"
         else
           puts "uploading #{file} to #{target_path}"
-          s3_object.write(content)
+          s3_object.put(body: content)
         end
       end
 

data/lib/pipely/live_pipeline.rb

@@ -1,4 +1,3 @@
-require 'pipely/fog_client'
 require 'pipely/runs_report'
 
 module Pipely
@@ -9,9 +8,8 @@ module Pipely
     def initialize(pipeline_id)
       @pipeline_id = pipeline_id
 
-      client = FogClient.new(pipeline_id)
-      @definition_json = client.definition
-      @task_states_by_scheduled_start = client.task_states_by_scheduled_start
+      @definition_json = definition(pipeline_id)
+      @task_states_by_scheduled_start = task_states_by_scheduled_start
 
       unless @definition_json
         raise "No definition found for #{client.pipeline_id}"
@@ -38,7 +36,11 @@ module Pipely
       end
     end
 
-  private
+    private
+
+    def data_pipeline
+      @data_pipeline ||= Aws::DataPipeline::Client.new
+    end
 
     def render_graph(start, task_states, output_path)
       utc_time = Time.now.to_i
@@ -50,5 +52,81 @@ module Pipely
       Pipely.draw(@definition_json, filename, task_states)
     end
 
+    def definition(pipeline_id)
+      objects = data_pipeline.get_pipeline_definition(pipeline_id: pipeline_id)
+      { objects: flatten_pipeline_objects(objects.pipeline_objects) }.to_json
+    end
+
+    def task_states_by_scheduled_start
+      task_states_by_scheduled_start = {}
+
+      all_instances.each do |pipeline_object|
+        component_id = status = scheduled_start = nil
+
+        pipeline_object.fields.each do |field|
+          case field.key
+          when '@componentParent'
+            component_id = field.ref_value
+          when '@status'
+            status = field.string_value
+          when '@scheduledStartTime'
+            scheduled_start = field.string_value
+          end
+        end
+
+        task_states_by_scheduled_start[scheduled_start] ||= {}
+        task_states_by_scheduled_start[scheduled_start][component_id] = {
+          execution_state: status
+        }
+      end
+
+      task_states_by_scheduled_start
+    end
+
+    def all_instances
+      result = {}
+      pipeline_objects = []
+
+      begin
+        result = data_pipeline.query_objects(
+          pipeline_id: pipeline_id,
+          sphere: "INSTANCE",
+          marker: result.marker,
+        )
+
+        instance_details = data_pipeline.describe_objects(
+          pipeline_id: pipeline_id,
+          object_ids: result.ids
+        )
+
+        data_pipeline.describe_objects(pipeline_id, result['ids'])
+        pipeline_objects += instance_details.pipeline_objects
+
+      end while (result.has_more_results && result.marker)
+
+      pipeline_objects
+    end
+
+
+    def flatten_pipeline_objects(objects)
+      objects.each_with_object([]) do |object, result|
+        h = {
+          id: object.id,
+          name: object.name,
+        }
+
+        object.fields.each do |field|
+          k = field.key
+          if field.ref_value
+            h[k] ||= []
+            h[k] << { ref: field.ref_value }
+          else
+            h[k] = field.string_value
+          end
+        end
+
+        result << h
+      end
+    end
   end
 end

data/lib/pipely/s3_writer.rb

@@ -1,4 +1,4 @@
-require 'fog/storage'
+require 'aws-sdk'
 
 module Pipely
 
@@ -11,31 +11,11 @@ module Pipely
       @host, @path = uri.host, uri.path.gsub(/^\//,'')
     end
 
-    def directory
-      directory = storage.directories.detect{ |d| d.key == @host }
-
-      directory or raise("Couldn't find S3 bucket '#{@host}'")
-    end
-
     def write(content)
-      remote_file = directory.files.create({
-        :key => @path,
-        :body => content,
-        :public => true,
-      })
-
-      remote_file.public_url
+      s3_bucket = Aws::S3::Bucket.new(@host)
+      s3_object = s3_bucket.object(@path)
+      s3_object.put(body: content, acl: 'public')
+      s3_object.public_url
     end
-
-    private
-
-      def storage
-        Fog::Storage.new({ provider: 'AWS' })
-      rescue ArgumentError
-        $stderr.puts "#{self.class.name}: Falling back to IAM profile"
-        Fog::Storage.new({ provider: 'AWS', use_iam_profile: true })
-      end
-
   end
-
 end

data/lib/pipely/tasks/upload_pipeline_as_gem.rb

@@ -1,6 +1,6 @@
 require 'rake'
 require 'rake/tasklib'
-require 'aws'
+require 'aws-sdk'
 require 'erubis'
 require 'pipely/deploy/bootstrap'
 
@@ -59,7 +59,7 @@ module Pipely
 
       private
       def s3_bucket
-          @s3_bucket ||= AWS::S3.new.buckets[@bucket_name]
+          @s3_bucket ||= Aws::S3::Bucket.new(@bucket_name)
       end
 
       def upload_gems

data/lib/pipely/tasks/upload_steps.rb

@@ -65,8 +65,8 @@ module Pipely
     private
 
       def with_bucket
-        s3 = AWS::S3.new
-        bucket = s3.buckets[s3_bucket_name]
+        s3 = Aws::S3::Resource.new
+        bucket = s3.bucket(s3_bucket_name)
 
         if bucket.exists?
           yield(bucket)

data/lib/pipely/version.rb

@@ -1,3 +1,3 @@
 module Pipely
-  VERSION = '0.13.0' unless defined?(::Pipely::VERSION)
+  VERSION = '0.14.0' unless defined?(::Pipely::VERSION)
 end

data/spec/lib/pipely/deploy/client_spec.rb

@@ -31,18 +31,18 @@ describe Pipely::Deploy::Client do
   describe '#create_pipeline' do
     let(:pipeline_name) { 'NewPipeline' }
     let(:pipeline_id) { 123 }
-    let(:created_pipeline) { double(:created_pipeline, id: pipeline_id) }
+    let(:created_pipeline) do
+      double(:created_pipeline, pipeline_id: pipeline_id)
+    end
     let(:definition) { "Pipeline ID: 123" }
 
-    let(:data_pipelines) { subject.instance_variable_get(:@data_pipelines) }
     let(:aws) { subject.instance_variable_get(:@aws) }
 
     it 'gets the definition from the block' do
-      data_pipelines.stub_chain(:pipelines, :create)
-        .and_return(created_pipeline)
 
       Pipely::Deploy::JSONDefinition.should_receive(:parse).with(definition)
 
+      aws.should_receive(:create_pipeline).and_return(created_pipeline)
       aws.should_receive(:put_pipeline_definition).and_return({})
       aws.should_receive(:activate_pipeline)
       subject.create_pipeline(pipeline_name, nil) do |pipeline_id|

data/spec/lib/pipely/deploy/s3_uploader_spec.rb

@@ -8,10 +8,9 @@ describe Pipely::Deploy::S3Uploader do
 
   subject { described_class.new(s3_bucket, 'test_path/gems') }
 
-  let(:s3_bucket) do
-     s3 = AWS::S3.new
-     s3.buckets['a-test-bucket']
-  end
+  let(:s3_object) { double(:s3_object) }
+  let(:bucket_name) { 'a-test-bucket' }
+  let(:s3_bucket) { double(:bucket, object: s3_object, name: bucket_name) }
 
   it "should have bucket name" do
     expect(subject.bucket_name).to eq('a-test-bucket')
@@ -22,12 +21,6 @@ describe Pipely::Deploy::S3Uploader do
   end
 
   describe "#upload(files)" do
-    let(:objects) { double(:objects) }
-
-    let(:s3_object) do
-      double('s3_object', write: nil, exists?: true, etag: 'mismatch')
-    end
-
     let(:files) do
       [
         Tempfile.new('packaged-gem1.gem').path,
@@ -35,17 +28,14 @@ describe Pipely::Deploy::S3Uploader do
       ]
     end
 
-    before do
-      allow(objects).to receive(:[]) { s3_object }
-      allow(s3_bucket).to receive(:objects) { objects }
-    end
-
     it 'uploads each file' do
+      allow(s3_object).to receive(:exists?).and_return(true)
+      allow(s3_object).to receive(:etag).and_return('mismatch')
       files.each do |file|
-        expect(objects).to receive(:[]).with(subject.s3_file_path(file))
+        expect(s3_bucket).to receive(:object).with(subject.s3_file_path(file))
       end
 
-      expect(s3_bucket).to receive(:objects).exactly(files.size).times
+      expect(s3_object).to receive(:put).exactly(files.size).times
 
       subject.upload(files)
     end

data/spec/spec_helper.rb

@@ -1,21 +1,15 @@
 require 'timecop'
 require 'aws-sdk'
-require 'fog'
 require 'rspec'
 require 'vcr'
 require 'pry'
 
 $LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib')
 
-AWS.config(
-  access_key_id: "xxx",
-  secret_access_key: "xxx"
-)
-
-Fog.credentials = {
-  aws_access_key_id: "xxx",
-  aws_secret_access_key: "xxx"
-}
+Aws.config.update({
+  region: 'us-east-1',
+  credentials: Aws::Credentials.new('xxx', 'xxx'),
+})
 
 VCR.configure do |c|
   c.allow_http_connections_when_no_cassette = true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pipely
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.14.0
 platform: ruby
 authors:
 - Matt Gillooly
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-27 00:00:00.000000000 Z
+date: 2015-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-graphviz
@@ -52,34 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.0.0
-- !ruby/object:Gem::Dependency
-  name: fog
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.23'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.23'
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.48'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.48'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: unf
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.0
+- !ruby/object:Gem::Dependency
+  name: safe_yaml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.4
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -259,7 +259,6 @@ files:
 - lib/pipely/deploy/client.rb
 - lib/pipely/deploy/json_definition.rb
 - lib/pipely/deploy/s3_uploader.rb
-- lib/pipely/fog_client.rb
 - lib/pipely/graph_builder.rb
 - lib/pipely/live_pipeline.rb
 - lib/pipely/options.rb
@@ -332,7 +331,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Generate dependency graphs from pipeline definitions.

data/lib/pipely/fog_client.rb

@@ -1,96 +0,0 @@
-require 'fog'
-
-module Pipely
-
-  # Uses Fog to communicate with the AWS Data Pipeline service
-  class FogClient < Struct.new(:pipeline_id)
-
-    def definition
-      objects = data_pipeline.get_pipeline_definition(pipeline_id)
-
-      flattened_objects = []
-
-      objects['pipelineObjects'].each do |object|
-        h = {
-          id: object['id'],
-          name: object['name'],
-        }
-
-        object['fields'].each do |field|
-          k = field['key']
-          if field['refValue']
-            h[k] ||= []
-            h[k] << { ref: field['refValue'] }
-          else
-            h[k] = field['stringValue']
-          end
-        end
-
-        flattened_objects << h
-      end
-
-      { objects: flattened_objects }.to_json
-    end
-
-    def task_states_by_scheduled_start
-      task_states_by_scheduled_start = {}
-
-      all_instances.each do |pipeline_object|
-        component_id = status = scheduled_start = nil
-
-        pipeline_object['fields'].each do |field|
-          case field['key']
-          when '@componentParent'
-            component_id = field['refValue']
-          when '@status'
-            status = field['stringValue']
-          when '@scheduledStartTime'
-            scheduled_start = field['stringValue']
-          end
-        end
-
-        task_states_by_scheduled_start[scheduled_start] ||= {}
-        task_states_by_scheduled_start[scheduled_start][component_id] = {
-          execution_state: status
-        }
-      end
-
-      task_states_by_scheduled_start
-    end
-
-  private
-
-    def all_instances
-      c = data_pipeline
-
-      result = {}
-      pipeline_objects = []
-
-      begin
-        if result['marker']
-          marker = JSON.parse(result['marker'])['primary']
-        end
-
-        result = c.query_objects(
-          pipeline_id,
-          'INSTANCE',
-          marker: result['marker']
-        )
-
-        instance_details = c.describe_objects(pipeline_id, result['ids'])
-        pipeline_objects += instance_details['pipelineObjects']
-
-      end while (result['hasMoreResults'] && result['marker'])
-
-      pipeline_objects
-    end
-
-    def data_pipeline
-      Fog::AWS::DataPipeline.new
-    rescue ArgumentError
-      $stderr.puts "#{self.class.name}: Falling back to IAM profile"
-      Fog::AWS::DataPipeline.new(use_iam_profile: true)
-    end
-
-  end
-end