aw_datapipe 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c7db5771c39e741d11c07927b1702e5c0db4372e
-  data.tar.gz: 8e005911037c8bc04e00715cdc41dc49c068c7ff
+  metadata.gz: c983384c36d63b6a8cacc019d757a34159e18202
+  data.tar.gz: b5af0af6d9157e7083a9645cf547d3cf974d66a1
 SHA512:
-  metadata.gz: e84ae673cb43f5819d231834e8ffd66b538ed16c50c8e82f5cac8673ae5655fa66adabc1efd45246f2477e58c3d6b4d2893b83fa61de072b474eed4416e958d4
-  data.tar.gz: 21740c23b42b4b01143827274fb1e7ec3508be1739cd8a0f39d14783a52007674b2c632c06c1d095169932bbe4fc09e94e51bc14556ee61f04b764833bab4a74
+  metadata.gz: dc704568621731f24931f5ace8d66144ec4ee3009e5633543c59b499022b8510e76f36ec5d7d518e414a9cc60739b75480b173fbacf6944939825294542db962
+  data.tar.gz: 4dac32dfe96dfa8ba2dce0246c446cf6f85215cfc134d18d6a056606f3de25edca592af16c1866d789ff7ebcebc93560b69660ed6b25007b7df95a6f147e5129

data/CHANGELOG.md

@@ -1,7 +1,10 @@
 # CHANGELOG
 
 ## TODO
-- Add DSL.
+- Updating.
+  - http://stackoverflow.com/questions/31188739/how-to-automate-the-updating-editing-of-amazon-data-pipeline
+  - http://docs.aws.amazon.com/datapipeline/latest/DeveloperGuide/dp-manage-pipeline-modify-console.html
+- Add DSL using clean room.
 - Add Thor based utility instead of console for downloads.
 - Generate separate SQL script files.
 - Codecov.
@@ -9,7 +12,11 @@
 - Rubydocs.
 - AWS labs examples converted to DSL.
 
-## 0.2.0
+## 0.2.1 - 2017-05-16
+- Allow either S3 directory\_path or file\_path.
+- Add ShellCommandActivity.
+
+## 0.2.0 - 2017-05-06
 - [FIX] Handle array values, e.g. 'securityGroups' => ['group1', 'group2']
 - Separate SourceWriter class.
 - Simplify build vs. new.

data/README.md

@@ -50,12 +50,16 @@ A live AWS account with a sample pipeline is required to run the remote tests.
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`. 
 
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/varyonic/aw_datapipe.
 
+## Releasing
+
+To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/aw_datapipe.gemspec

@@ -12,7 +12,6 @@ Gem::Specification.new do |spec|
   spec.summary       = "Unofficial ruby wrapper for the AWS SDK Data Pipeline API."
   spec.description   = "Unofficial domain specific ruby wrapper for the AWS SDK Data Pipeline API."
   spec.homepage      = "http://github.com/varyonic/aw_datapipe"
-  spec.date          = "2017-05-02"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|

data/lib/aw_datapipe/pipeline_object.rb

@@ -60,7 +60,8 @@ module AwDatapipe
   Schedule = PipelineObject.new(:period, :start_date_time)
 
   Ec2Resource = PipelineObject.new(:action_on_task_failure, :instance_type, :security_group_ids, :subnet_id, :terminate_after)
-  S3DataNode = PipelineObject.new(:directory_path)
+  S3DataNode = PipelineObject.new(:directory_path, :file_path)
+  ShellCommandActivity = PipelineObject.new(:command, :input, :output, :stage, :runs_on)
 
   JdbcDatabase = PipelineObject.new(:_password, :connection_string, :jdbc_driver_class, :username)
   SqlDataNode = PipelineObject.new(:database, :select_query, :table)

data/lib/aw_datapipe/pipeline_serializer.rb

@@ -80,10 +80,11 @@ module AwDatapipe
           { key: camelized_key, ref_value: camelize(value.id) }
         when Array
           value.map { |v| Hash[key: camelized_key, string_value: v] }
+        when NilClass
         else
           { key: camelized_key, string_value: value }
         end
-      end.flatten
+      end.flatten.compact
     end
 
     # Convert string to a rubyish variable name.

data/lib/aw_datapipe/version.rb

@@ -1,3 +1,3 @@
 module AwDatapipe
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aw_datapipe
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Piers Chambers
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-02 00:00:00.000000000 Z
+date: 2017-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport