aw_datapipe 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fe971d0b5fa7e5a8558d43e7e0940e634d3bbfeb
-  data.tar.gz: 9ccb16f29ef04f12ff5fdbc23d042e86d762393a
+  metadata.gz: c7db5771c39e741d11c07927b1702e5c0db4372e
+  data.tar.gz: 8e005911037c8bc04e00715cdc41dc49c068c7ff
 SHA512:
-  metadata.gz: c26e3f06941dabdda94fa17d8dabf0edd623820b7ce8579efd7f78f51ef32c9e2f085a68c7fd022b41a11431d7ca8fc026a2838e28ec92711bcda06f85293c38
-  data.tar.gz: d7758fde384e2d782805b0b9b0c3fb2e0c3094f31a6d4d2863ac8e49664f211567054fd35e76cb260c68e139998dbdfef9f12ab69438b6180f345aface21d2f5
+  metadata.gz: e84ae673cb43f5819d231834e8ffd66b538ed16c50c8e82f5cac8673ae5655fa66adabc1efd45246f2477e58c3d6b4d2893b83fa61de072b474eed4416e958d4
+  data.tar.gz: 21740c23b42b4b01143827274fb1e7ec3508be1739cd8a0f39d14783a52007674b2c632c06c1d095169932bbe4fc09e94e51bc14556ee61f04b764833bab4a74

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.2.2
+before_install: gem install bundler -v 1.14

data/CHANGELOG.md

@@ -0,0 +1,21 @@
+# CHANGELOG
+
+## TODO
+- Add DSL.
+- Add Thor based utility instead of console for downloads.
+- Generate separate SQL script files.
+- Codecov.
+- CodeClimate.
+- Rubydocs.
+- AWS labs examples converted to DSL.
+
+## 0.2.0
+- [FIX] Handle array values, e.g. 'securityGroups' => ['group1', 'group2']
+- Separate SourceWriter class.
+- Simplify build vs. new.
+- Default id value based on pipeline object name.
+- Unit tests, eg. PipelineSerializer.
+- Travis.
+
+## 0.1.0 - 2017-05-03
+- Proof of concept: get and put pipeline definitions, marshal and unmarshal into ruby objects and generate working update script output.

data/README.md

@@ -1,4 +1,6 @@
 # AW Datapipe
+[![Gem Version](https://badge.fury.io/rb/aw_datapipe.png)](https://badge.fury.io/rb/aw_datapipe)
+[![Build Status](https://travis-ci.org/varyonic/aw_datapipe.png?branch=master)](https://travis-ci.org/varyonic/aw_datapipe)
 
 AW Datapipe is an unofficial domain specific ruby wrapper for the 
 [AWS SDK](http://www.rubydoc.info/github/aws/aws-sdk-ruby) Data Pipeline API.
@@ -30,8 +32,9 @@ Configure credentials for AWS SDK.
 export AWS_ACCESS_KEY_ID=AKIA****************
 export AWS_SECRET_ACCESS_KEY=********************************
 ```
-Use bin/console to download a pipeline definition as ruby instead of JSON.
+Use a ruby console (e.g. irb) to download a pipeline definition as ruby instead of JSON.
 ```ruby
+require 'aw_datapipe'
 pipelines = AwDatapipe::Session.new
 pipelines.download_definition 'df-***************', 'tmp/pipeline-definition.rb'
 ```
@@ -43,7 +46,7 @@ bundle exec ruby tmp/pipeline-definition.rb
 ```
 ## Development
 
-A live AWS account with a sample pipeline is required to run the tests.
+A live AWS account with a sample pipeline is required to run the remote tests.
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
@@ -53,7 +56,6 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/varyonic/aw_datapipe.
 
-
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -1,10 +1,22 @@
 require "bundler/gem_tasks"
 require "rake/testtask"
 
-Rake::TestTask.new(:test) do |t|
-  t.libs << "test"
-  t.libs << "lib"
-  t.test_files = FileList['test/**/*_test.rb']
-end
+desc "Run the unit test suite"
+task :default => 'test:units'
+task :test => 'test:units'
+
+namespace :test do
+  Rake::TestTask.new(:units) do |t|
+    t.pattern = 'test/unit/**/*_test.rb'
+    t.ruby_opts << '-rubygems -w'
+    t.libs << 'test'
+    t.verbose = true
+  end
 
-task :default => :test
+  Rake::TestTask.new(:remote) do |t|
+    t.pattern = 'test/remote/**/*_test.rb'
+    t.ruby_opts << '-rubygems -w'
+    t.libs << 'test'
+    t.verbose = true
+  end
+end

data/aw_datapipe.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency("activesupport")
+  spec.add_dependency("activesupport", ">= 3")
   spec.add_dependency("aws-sdk", ['~> 2'])
 
   spec.add_development_dependency "bundler", "~> 1.14"

data/lib/aw_datapipe.rb

@@ -1,3 +1,4 @@
+require 'active_support/core_ext/module/delegation'
 require 'active_support/inflector' # String#underscore
 require 'aws-sdk'
 
@@ -7,4 +8,5 @@ require 'aw_datapipe/pipeline_object'
 require 'aw_datapipe/pipeline_serializer'
 require 'aw_datapipe/object_hash'
 require 'aw_datapipe/session'
+require 'aw_datapipe/source_writer'
 require 'aw_datapipe/version'

data/lib/aw_datapipe/object_hash.rb

@@ -13,13 +13,5 @@ module AwDatapipe
       self
     end
     alias_method :<<, :append
-
-    def append_with_dependents(*objects)
-      objects.each do |object|
-        object.dependencies.each { |dep| self.append dep }
-        self.append object
-      end
-      self
-    end
   end
 end

data/lib/aw_datapipe/pipeline.rb

@@ -15,7 +15,8 @@ module AwDatapipe
     end
 
     def self.build(config, activities, parameter_metadata, parameter_values)
-      new([], parameter_metadata, parameter_values).tap { |p| p.objects.append_with_dependents(config, *activities) }
+      objects = [config, *activities].map { |obj| obj.dependencies.append(obj) }.flatten
+      new(objects, parameter_metadata, parameter_values)
     end
 
     def configuration
@@ -32,58 +33,8 @@ module AwDatapipe
       objects.values.map(&:dependencies).flatten.uniq
     end
 
-    def source
-      [
-        header_source,
-        objects_source,
-        parameter_metadata_source,
-        parameter_values_source,
-        footer_source
-      ].join("\n")
-    end
-
     def write_source(pathname)
-      File.write(pathname, source)
-    end
-
-    protected
-
-    def footer_source
-      <<-EOF
-pipelines = AwDatapipe::Session.new
-pipeline = AwDatapipe::Pipeline.build(default, activities, parameter_metadata, parameter_values)
-pipeline.id = "#{id}"
-pipelines.save(pipeline)
-      EOF
-    end
-
-    def header_source
-      <<-EOF
-# Generated by aw_datapipe download_definition of #{id}
-require 'aw_datapipe'
-      EOF
-    end
-
-    def objects_source
-      object_ids = objects.keys
-      unreferenced_object_ids = object_ids - referenced_object_ids
-
-      s = referenced_object_ids.map { |id| "#{id} = #{objects[id].source}" }.join("\n\n")
-      s << "\n\nactivities = [\n"
-      s << unreferenced_object_ids.map { |id| "  #{objects[id].source(2)}" }.join(",\n")
-      s << "\n]"
-    end
-
-    def parameter_values_source
-      "parameter_values = {\n  " << parameter_values.sort.map do |key, value|
-        "\"#{key}\" => #{value.inspect}"
-      end.join(",\n  ") << "\n}\n"
-    end
-
-    def parameter_metadata_source
-      "parameter_metadata = {\n  " << parameter_metadata.sort.map do |key, value|
-        "\"#{key}\" => #{value.source}"
-      end.join(",\n  ") << "\n}\n"
+      SourceWriter.call(self, pathname)
     end
   end
 end

data/lib/aw_datapipe/pipeline_object.rb

@@ -1,17 +1,23 @@
 module AwDatapipe
   class PipelineObject < Struct
+    attr_reader :id
+    attr_accessor :name
     attr_accessor :pipeline
 
     def self.build(params)
-      new.tap do |struct|
-        params.each_pair { |k, v| struct.send "#{k}=", v }
-      end
+      new(params)
+    end
+
+    def initialize(params)
+      @id = params.delete(:id) || params[:name].underscore.to_sym ||
+        raise(ArgumentError, ":id or :name required: #{params.inspect}")
+      params.each_pair { |k, v| send "#{k}=", v }
     end
 
     # Iterates through struct members, recursively collecting any PipelineObjects.
     # Recursion ensures dependencies sorted before dependents.
     def dependencies
-      (members - [:id]).each_with_object([]) do |attr_name, depends|
+      members.each_with_object([]) do |attr_name, depends|
         value = send(attr_name)
         value = pipeline.objects.fetch(value) if value.is_a?(Symbol)
         depends << value.dependencies << value if value.is_a?(PipelineObject)
@@ -22,14 +28,18 @@ module AwDatapipe
       self.class.name.split('::').last
     end
 
+    def inspect
+      "#<#{demodulized_class_name} #{to_hash}>"
+    end
+
     def to_hash
-      Hash[each_pair.to_a]
+      Hash[each_pair.to_a].merge(id: id, name: name)
     end
 
     def source(indent_level = 1)
-      "#{self.class.name}.build(\n" << indent(indent_level) << members.map do |member|
-        member_source(member)
-      end.join(",\n" << indent(indent_level)) << ")"
+      "#{self.class.name}.build(" << [:id, :name, *members].map do |member|
+        "\n" << indent(indent_level) << member_source(member)
+      end.join(",") << ")"
     end
 
     protected
@@ -46,18 +56,18 @@ module AwDatapipe
     end
   end
 
-  Configuration = PipelineObject.new(:failure_and_rerun_mode, :id, :name, :pipeline_log_uri, :resource_role, :role, :schedule, :schedule_type)
-  Schedule = PipelineObject.new(:id, :name, :period, :start_date_time)
+  Configuration = PipelineObject.new(:failure_and_rerun_mode, :pipeline_log_uri, :resource_role, :role, :schedule, :schedule_type)
+  Schedule = PipelineObject.new(:period, :start_date_time)
 
-  Ec2Resource = PipelineObject.new(:action_on_task_failure, :id, :instance_type, :name, :security_group_ids, :subnet_id, :terminate_after)
-  S3DataNode = PipelineObject.new(:directory_path, :id, :name)
+  Ec2Resource = PipelineObject.new(:action_on_task_failure, :instance_type, :security_group_ids, :subnet_id, :terminate_after)
+  S3DataNode = PipelineObject.new(:directory_path)
 
-  JdbcDatabase = PipelineObject.new(:_password, :connection_string, :id, :jdbc_driver_class, :name, :username)
-  SqlDataNode = PipelineObject.new(:database, :id, :name, :select_query, :table)
-  CopyActivity = PipelineObject.new(:id, :input, :name, :output, :runs_on)
+  JdbcDatabase = PipelineObject.new(:_password, :connection_string, :jdbc_driver_class, :username)
+  SqlDataNode = PipelineObject.new(:database, :select_query, :table)
+  CopyActivity = PipelineObject.new(:input, :output, :runs_on)
 
-  RedshiftDatabase = PipelineObject.new(:_password, :connection_string, :database_name, :id, :name, :username)
-  RedshiftDataNode = PipelineObject.new(:create_table_sql, :database, :id, :name, :primary_keys, :schema_name, :table_name)
-  RedshiftCopyActivity = PipelineObject.new(:id, :input, :insert_mode, :name, :output, :runs_on)
+  RedshiftDatabase = PipelineObject.new(:_password, :connection_string, :database_name, :username)
+  RedshiftDataNode = PipelineObject.new(:create_table_sql, :database, :primary_keys, :schema_name, :table_name)
+  RedshiftCopyActivity = PipelineObject.new(:input, :insert_mode, :output, :runs_on)
 
 end

data/lib/aw_datapipe/pipeline_serializer.rb

@@ -12,9 +12,9 @@ module AwDatapipe
 
     def unmarshal(aws_definition)
       # pipeline.aws_definition = aws_definition # for troubleshooting
-      objects = unmarshal_pipeline_objects(aws_definition)
-      parameter_metadata = unmarshal_parameter_objects(aws_definition)
-      parameter_values = unmarshal_parameter_values(aws_definition)
+      objects = unmarshal_pipeline_objects(aws_definition.pipeline_objects)
+      parameter_metadata = unmarshal_parameter_objects(aws_definition.parameter_objects)
+      parameter_values = unmarshal_parameter_values(aws_definition.parameter_values)
 
       Pipeline.new(objects, parameter_metadata, parameter_values)
     end
@@ -43,8 +43,8 @@ module AwDatapipe
     end
 
     # @return Array PipelineObject subclass instance.
-    def unmarshal_pipeline_objects(aws_definition)
-      aws_definition.pipeline_objects.map do |aws_struct|
+    def unmarshal_pipeline_objects(pipeline_objects)
+      pipeline_objects.map do |aws_struct|
         unmarshal_pipeline_object(aws_struct)
       end
     end
@@ -57,23 +57,33 @@ module AwDatapipe
 
       klass = AwDatapipe.const_defined?(type, false) ?
         AwDatapipe.const_get(type, false) :
-        AwDatapipe.const_set(type, PipelineObject.new(*attributes.keys.sort))
+        AwDatapipe.const_set(type, PipelineObject.new(*(attributes.keys - [:id, :name])))
 
-      klass.new(*attributes.sort.map(&:last)) # pass values sorted by keys
+      klass.new(attributes)
     end
 
     def fields_to_hash(fields)
       fields.each_with_object({}) do |field, hash|
-        hash[symbolize field.key] = field.string_value || field.ref_value.underscore.to_sym
+        if hash[symbolize field.key]
+          hash[symbolize field.key] = Array(hash[symbolize field.key]) << field.string_value
+        else
+          hash[symbolize field.key] = field.string_value || field.ref_value.underscore.to_sym
+        end
       end
     end
 
     def hash_to_fields(hash)
-      hash.keys.map do |key|
-        PipelineObject === hash[key] ?
-          { key: camelize(key, :lower), ref_value: camelize(hash[key].id) } :
-          { key: camelize(key, :lower), string_value: hash[key] }
-      end
+      hash.each_pair.map do |key, value|
+        camelized_key = camelize(key, :lower)
+        case value
+        when PipelineObject
+          { key: camelized_key, ref_value: camelize(value.id) }
+        when Array
+          value.map { |v| Hash[key: camelized_key, string_value: v] }
+        else
+          { key: camelized_key, string_value: value }
+        end
+      end.flatten
     end
 
     # Convert string to a rubyish variable name.
@@ -98,8 +108,8 @@ module AwDatapipe
       { id: key, attributes: out }
     end
 
-    def unmarshal_parameter_objects(aws_definition)
-      aws_definition.parameter_objects.each_with_object({}) do |object, hash|
+    def unmarshal_parameter_objects(parameter_objects)
+      parameter_objects.each_with_object({}) do |object, hash|
         klass = ParameterMetadata.new(*object.attributes.map(&:key).map(&:to_sym))
         hash[object.id] = object.attributes.each_with_object(klass.new) do |attribute, struct|
           struct.send "#{attribute.key}=", attribute.string_value
@@ -110,14 +120,22 @@ module AwDatapipe
     def marshal_parameter_values(parameter_values)
       out = []
       parameter_values.each_pair do |id, value|
-        out << { id: id, string_value: value }
+        if value.respond_to?(:each)
+          value.each { |v| out << { id: id, string_value: v } }
+        else
+          out << { id: id, string_value: value }
+        end
       end
       out
     end
 
-    def unmarshal_parameter_values(aws_definition)
-      aws_definition.parameter_values.each_with_object({}) do |value, hash|
-        hash[value.id] = value.string_value
+    def unmarshal_parameter_values(parameter_values)
+      parameter_values.each_with_object({}) do |value, hash|
+        if hash[value.id]
+          hash[value.id] = Array(hash[value.id]) << value.string_value
+        else
+          hash[value.id] = value.string_value
+        end
       end
     end
   end

data/lib/aw_datapipe/source_writer.rb

@@ -0,0 +1,68 @@
+module AwDatapipe
+  # Writes a Pipeline definition into a ruby script.
+  class SourceWriter
+    def self.call(pipeline, pathname)
+      new(pipeline).call(pathname)
+    end
+
+    attr_reader :pipeline
+    delegate :objects, :parameter_values, :parameter_metadata, :referenced_object_ids, to: :pipeline
+
+    def initialize(pipeline)
+      @pipeline = pipeline
+    end
+
+    def call(pathname)
+      File.write(pathname, source)
+    end
+
+    def source
+      [
+        header_source,
+        objects_source,
+        parameter_metadata_source,
+        parameter_values_source,
+        footer_source
+      ].join("\n")
+    end
+
+    protected
+
+    def header_source
+      <<-EOF
+# Generated by aw_datapipe download_definition of #{pipeline.id}
+require 'aw_datapipe'
+      EOF
+    end
+
+    def objects_source
+      object_ids = objects.keys
+      unreferenced_object_ids = object_ids - referenced_object_ids
+      s = referenced_object_ids.map { |id| "#{id} = #{objects[id].source}" }.join("\n\n")
+      s << "\n\nactivities = [\n"
+      s << unreferenced_object_ids.map { |id| "  #{objects[id].source(2)}" }.join(",\n")
+      s << "\n]"
+    end
+
+    def parameter_metadata_source
+      "parameter_metadata = {\n  " << parameter_metadata.sort.map do |key, value|
+        "\"#{key}\" => #{value.source}"
+      end.join(",\n  ") << "\n}\n"
+    end
+
+    def parameter_values_source
+      "parameter_values = {\n  " << parameter_values.sort.map do |key, value|
+        "\"#{key}\" => #{value.inspect}"
+      end.join(",\n  ") << "\n}\n"
+    end
+
+    def footer_source
+      <<-EOF
+pipelines = AwDatapipe::Session.new
+pipeline = AwDatapipe::Pipeline.build(default, activities, parameter_metadata, parameter_values)
+pipeline.id = "#{pipeline.id}"
+pipelines.save(pipeline)
+      EOF
+    end
+  end
+end

data/lib/aw_datapipe/version.rb

@@ -1,3 +1,3 @@
 module AwDatapipe
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aw_datapipe
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Piers Chambers
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
@@ -89,6 +89,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -103,6 +105,7 @@ files:
 - lib/aw_datapipe/pipeline_object.rb
 - lib/aw_datapipe/pipeline_serializer.rb
 - lib/aw_datapipe/session.rb
+- lib/aw_datapipe/source_writer.rb
 - lib/aw_datapipe/version.rb
 homepage: http://github.com/varyonic/aw_datapipe
 licenses: