aw_datapipe 0.2.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 96dc0983a35a311d5063aa30cd7bc4f6baf92850
-  data.tar.gz: 463cd625f2bfcb21bc083012b1552eec143126ce
+  metadata.gz: bfef690d9ba54e7838eb2d46b45face330762037
+  data.tar.gz: 5cbba46fc31a079b9b4743b9a0439fb35259efbf
 SHA512:
-  metadata.gz: 9b186202282082464ea94090d938cd2ad0289531849cd52fe765f5229b9c3681ed2033b0452148b36f662dd2fa05bc2c5318bbf734102168b48442d9011e8f28
-  data.tar.gz: a196a481a168dcf3d15caba0c3c891cc544b8c5636cab994bb85eeaf0cc060cfcf52938236cf423c4304814fe7af1a6aee70332f4aebadbff8a101618e765e80
+  metadata.gz: 2f41ad60618c2def93ff91fd43b05d6163a135ba037b7e4afa782f16f0e886433a71311df35696eed1f4221057a4e70308b0e15061b51012a0674e621e28b710
+  data.tar.gz: 040f2327a07d062e3a6c47bef7770e8d8b6e9db2be690a5840631e9d8548172c3dbd3914abf34595acd9df7a90cf70303717d6e07ff30032dfef5b0eeee2c14d

data/CHANGELOG.md

@@ -4,7 +4,6 @@
 - Updating.
   - http://stackoverflow.com/questions/31188739/how-to-automate-the-updating-editing-of-amazon-data-pipeline
   - http://docs.aws.amazon.com/datapipeline/latest/DeveloperGuide/dp-manage-pipeline-modify-console.html
-- Add DSL using clean room.
 - Add Thor based utility instead of console for downloads.
 - Generate separate SQL script files.
 - Codecov.
@@ -12,6 +11,12 @@
 - Rubydocs.
 - AWS labs examples converted to DSL.
 
+## 0.3.0 - 2018-06-14
+- Use AWS SDK v3.
+- Separate appending activities from initializing pipeline with config.
+- Add helper methods to provide a more hierarchical builder API.
+- Provide default settings for a pipeline EC2 resource.
+
 ## 0.2.4 - 2017-05-24
 - Add ShellCommandActivity optional parameters.
 

data/aw_datapipe.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.2.2'
 
   spec.add_dependency("activesupport", ">= 3")
-  spec.add_dependency("aws-sdk", ['~> 2'])
+  spec.add_dependency("aws-sdk-datapipeline", '~> 1')
 
   spec.add_development_dependency "bundler", "~> 1.14"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/aw_datapipe/pipeline.rb

@@ -9,20 +9,64 @@ module AwDatapipe
 
     # objects [Array]
     def initialize(objects, parameter_metadata, parameter_values)
-      objects.each { |object| object.pipeline = self }
-      @objects ||= ObjectHash.new(*objects)
+      @objects = ObjectHash.new
+      append_objects_with_dependencies(objects)
       @parameter_metadata, @parameter_values = parameter_metadata, parameter_values
+      yield(self) if block_given?
     end
 
     def self.build(config, activities, parameter_metadata, parameter_values)
-      objects = [config, *activities].map { |obj| obj.dependencies.append(obj) }.flatten
-      new(objects, parameter_metadata, parameter_values)
+      new([config], parameter_metadata, parameter_values) do |pipeline|
+        pipeline.append_objects_with_dependencies(activities)
+      end
+    end
+
+    # @return [PipelineObject] appended object
+    def append_object(object)
+      object.pipeline = self
+      objects[object.id] = object
+    end
+
+    def append_object_with_dependencies(object)
+      [*object.dependencies, object].each(&method(:append_object))
+    end
+
+    # @return [Pipeline] self
+    def append_objects_with_dependencies(objects)
+      objects.each(&method(:append_object_with_dependencies))
+      self
     end
 
     def configuration
       objects.fetch(:default)
     end
 
+    def csv_data_format(params)
+      append_object CsvDataFormat.build(params)
+    end
+
+    # @param [Hash] params Various required and optional parameters.
+    # @option params [String] :name (default: 'Ec2Instance')
+    # @option params [String] :instance_type (default: 't1.micro')
+    # @option params [String] :subnet_id
+    # @option params [String] :security_group_ids
+    # @option params [String] :action_on_task_failure (default: 'terminate')
+    # @option params [String] :terminate_after (default: '2 Hours')
+    # 
+    # @return [Ec2Resource]
+    def ec2_resource(params)
+      append_object Ec2Resource.build(
+        params.reverse_merge(name: 'Ec2Instance', instance_type: 't1.micro', action_on_task_failure: 'terminate', terminate_after: '2 Hours'))
+    end
+
+    def jdbc_database(params)
+      append_object JdbcDatabase.build(params)
+    end
+
+    def s3_data_node(params)
+      append_object S3DataNode.build(params)
+    end
+
     def referenced_object_ids
       referenced_objects.map(&:id) << :default
     end

data/lib/aw_datapipe/pipeline_object.rb

@@ -59,7 +59,16 @@ module AwDatapipe
   Configuration = PipelineObject.new(:failure_and_rerun_mode, :pipeline_log_uri, :resource_role, :role, :schedule, :schedule_type)
   Schedule = PipelineObject.new(:period, :start_date_time)
 
-  Ec2Resource = PipelineObject.new(:action_on_task_failure, :instance_type, :security_group_ids, :subnet_id, :terminate_after)
+  Ec2Resource = PipelineObject.new(:action_on_task_failure, :instance_type, :security_group_ids, :subnet_id, :terminate_after) do
+    def copy_activity(params)
+      pipeline.append_object CopyActivity.build(params.merge(runs_on: self))
+    end
+
+    def shell_command_activity(params)
+      pipeline.append_object ShellCommandActivity.build(params.merge(runs_on: self))
+    end
+  end
+
   S3DataNode = PipelineObject.new(:directory_path, :data_format, :file_path)
   CsvDataFormat = PipelineObject.new(:column) do
     def type
@@ -68,7 +77,12 @@ module AwDatapipe
   end
   ShellCommandActivity = PipelineObject.new(:input, :output, :runs_on, :command, :script_argument, :script_uri, :stage)
 
-  JdbcDatabase = PipelineObject.new(:_password, :connection_string, :jdbc_driver_class, :username)
+  JdbcDatabase = PipelineObject.new(:_password, :connection_string, :jdbc_driver_class, :username) do
+    def sql_data_node(params)
+      pipeline.append_object SqlDataNode.build(params.merge(database: self))
+    end
+  end
+
   SqlDataNode = PipelineObject.new(:database, :select_query, :table)
   CopyActivity = PipelineObject.new(:input, :output, :runs_on)
 

data/lib/aw_datapipe/version.rb

@@ -1,3 +1,3 @@
 module AwDatapipe
-  VERSION = "0.2.4"
+  VERSION = '0.3.0'
 end

data/lib/aw_datapipe.rb

@@ -1,6 +1,6 @@
 require 'active_support/core_ext/module/delegation'
 require 'active_support/inflector' # String#underscore
-require 'aws-sdk'
+require 'aws-sdk-datapipeline'
 
 require 'aw_datapipe/parameter_metadata'
 require 'aw_datapipe/pipeline'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aw_datapipe
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.3.0
 platform: ruby
 authors:
 - Piers Chambers
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-24 00:00:00.000000000 Z
+date: 2018-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -25,19 +25,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk
+  name: aws-sdk-datapipeline
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -127,7 +127,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Unofficial ruby wrapper for the AWS SDK Data Pipeline API.