pipely 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5c3c702222fadfd21d7937f3bf45f72437cd2b60
-  data.tar.gz: 03ec23c642d82f9b246ed9d127f8623e22e9967b
+  metadata.gz: f340152a018c85f0b7528a12066e818836094117
+  data.tar.gz: f403ae809e469fefbd919ddcc796ac4a0e6cff0e
 SHA512:
-  metadata.gz: dcb8993c5e8957c62ceff5f0b6f3514a64ccc8760151fa19e22ecd344cb4cc187653844f88d841b2bc39081cfe4e8c72ca6e5b61cd4b665d3ba9f2016d180755
-  data.tar.gz: ae660c6b3da224af6854a1c0d0e2c3d90b5c5a69ace85b0573bc5d050948e0eaaf32116e8078e7450e42a4359061c71cb73d468f81a6dde933686eb14128a913
+  metadata.gz: 0a521348af1f0f35214b78b980e65fe88f058be2ef958730d90fb8fe0326801d2f9039703ea7586281cf6210dce9f70342f06cb003ad2f379c2db6a919b53f9a
+  data.tar.gz: d15ae4ae65db5d61c169b7563370ebe5474b549a9d281c52f94ea8d22560ff430f115a06f04d058de146d48ffe9edf6f3e798381493c116677f87a9d8fce484b

data/README.md

@@ -33,6 +33,9 @@ Coming soon.
     rake graph             # Graphs the full pipeline definition using Graphviz
     rake upload_steps      # Upload Data Pipeline steps to S3
 
+In order to run these tasks, you must have an aws-sdk credentials file.
+This can be created by running `aws cli configure`, as described [in the aws-sdk docs](http://docs.aws.amazon.com/AWSSdkDocsRuby/latest/DeveloperGuide/ruby-dg-setup.html#set-up-creds).
+
 ### Command-line Interface
 
 (If you used the Gemfile install, prefix the below commands with `bundle exec`.)

data/lib/pipely/definition.rb

@@ -69,4 +69,3 @@ module Pipely
   end
 
 end
-

data/lib/pipely/deploy/client.rb

@@ -1,4 +1,17 @@
+# Note: We are in the process of migrating from Fog to aws-sdk for communicating
+# with the Data Pipeline API.  aws-sdk offers several benefits, such as:
+#
+# * Built-in automated exponential back-off, to avoid hitting rate limits.
+# * Working pagination of ListPipelines responses.
+# * Authentication using IAM resource roles.
+# * Faster installation.
+#
+# On the downside, aws-sdk does not yet support tagging of pipelines.  We can
+# not yet port pipely entirely away from Fog until this is resolved, so we will
+# temporarily require both libraries.
+
 require 'fog'
+require 'aws-sdk'
 require 'logger'
 require 'tempfile'
 require 'uuidtools'
@@ -16,6 +29,7 @@ module Pipely
       def initialize(log=nil)
         @log = log || Logger.new(STDOUT)
         @data_pipelines = Fog::AWS::DataPipeline.new
+        @aws = AWS::DataPipeline.new.client
       end
 
       def deploy_pipeline(pipeline_basename, definition)
@@ -51,15 +65,17 @@ module Pipely
 
       def existing_pipelines(pipeline_name)
         ids = []
+        marker = nil
 
         begin
-          result = Fog::AWS[:data_pipeline].list_pipelines
+          options = marker ? { marker: marker } : {}
+          result = @aws.list_pipelines(options)
 
-          ids += result['pipelineIdList'].
-                   select { |p| p['name'] == pipeline_name }.
-                   map { |p| p['id'] }
+          ids += result[:pipeline_id_list].
+                   select { |p| p[:name] == pipeline_name }.
+                   map { |p| p[:id] }
 
-        end while (result['hasMoreResults'] && result['marker'])
+        end while (result[:has_more_results] && marker = result[:marker])
 
         ids
       end

data/lib/pipely/tasks/definition.rb

@@ -1,6 +1,8 @@
 require 'rake'
 require 'rake/tasklib'
 require 'pipely'
+require 'json'
+require 'fileutils'
 
 module Pipely
   module Tasks
@@ -55,8 +57,14 @@ module Pipely
       def run_task(verbose)
         puts "Generating #{target_filename}" if verbose
 
+        json = definition.to_json
+
+        unless ENV['UGLY']
+          json = JSON.pretty_generate(JSON.parse(json))
+        end
+
         File.open(target_filename, 'w') do |file|
-          file.write(definition.to_json)
+          file.write(json)
         end
       end
 

data/lib/pipely/version.rb

@@ -1,3 +1,3 @@
 module Pipely
-  VERSION = "0.5.0" unless defined?(::Pipely::VERSION)
+  VERSION = "0.6.0" unless defined?(::Pipely::VERSION)
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pipely
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Matt Gillooly
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-17 00:00:00.000000000 Z
+date: 2014-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-graphviz
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.23.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.48'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.48'
 - !ruby/object:Gem::Dependency
   name: unf
   requirement: !ruby/object:Gem::Requirement