aw_datapipe 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: fe971d0b5fa7e5a8558d43e7e0940e634d3bbfeb
+  data.tar.gz: 9ccb16f29ef04f12ff5fdbc23d042e86d762393a
+SHA512:
+  metadata.gz: c26e3f06941dabdda94fa17d8dabf0edd623820b7ce8579efd7f78f51ef32c9e2f085a68c7fd022b41a11431d7ca8fc026a2838e28ec92711bcda06f85293c38
+  data.tar.gz: d7758fde384e2d782805b0b9b0c3fb2e0c3094f31a6d4d2863ac8e49664f211567054fd35e76cb260c68e139998dbdfef9f12ab69438b6180f345aface21d2f5

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in aw_datapipe.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2017 Piers Chambers
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,59 @@
+# AW Datapipe
+
+AW Datapipe is an unofficial domain specific ruby wrapper for the 
+[AWS SDK](http://www.rubydoc.info/github/aws/aws-sdk-ruby) Data Pipeline API.
+
+The primary goal is to support ruby scripts for creating and updating 
+pipeline definitions.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'aw_datapipe'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install aw_datapipe
+
+## Usage
+
+Configure credentials for AWS SDK.
+
+```sh
+export AWS_ACCESS_KEY_ID=AKIA****************
+export AWS_SECRET_ACCESS_KEY=********************************
+```
+Use bin/console to download a pipeline definition as ruby instead of JSON.
+```ruby
+pipelines = AwDatapipe::Session.new
+pipelines.download_definition 'df-***************', 'tmp/pipeline-definition.rb'
+```
+
+The generated script can be checked into version control, modified and executed
+to update the pipeline definition.
+```sh
+bundle exec ruby tmp/pipeline-definition.rb
+```
+## Development
+
+A live AWS account with a sample pipeline is required to run the tests.
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/varyonic/aw_datapipe.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/aw_datapipe.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'aw_datapipe/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "aw_datapipe"
+  spec.version       = AwDatapipe::VERSION
+  spec.authors       = ["Piers Chambers"]
+  spec.email         = ["piers@varyonic.com"]
+
+  spec.summary       = "Unofficial ruby wrapper for the AWS SDK Data Pipeline API."
+  spec.description   = "Unofficial domain specific ruby wrapper for the AWS SDK Data Pipeline API."
+  spec.homepage      = "http://github.com/varyonic/aw_datapipe"
+  spec.date          = "2017-05-02"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency("activesupport")
+  spec.add_dependency("aws-sdk", ['~> 2'])
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "aw_datapipe"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/aw_datapipe.rb

@@ -0,0 +1,10 @@
+require 'active_support/inflector' # String#underscore
+require 'aws-sdk'
+
+require 'aw_datapipe/parameter_metadata'
+require 'aw_datapipe/pipeline'
+require 'aw_datapipe/pipeline_object'
+require 'aw_datapipe/pipeline_serializer'
+require 'aw_datapipe/object_hash'
+require 'aw_datapipe/session'
+require 'aw_datapipe/version'

data/lib/aw_datapipe/object_hash.rb

@@ -0,0 +1,25 @@
+module AwDatapipe
+  # A symbol table implemeted as a hash of objects keyed by their ids.
+  class ObjectHash < Hash
+    def initialize(*objects)
+      super()
+      self.append(*objects)
+    end
+
+    # Adds PipelineObjects to the symbol table
+    # along with any of their dependencies.
+    def append(*objects)
+      objects.each { |object| self[object.id] = object }
+      self
+    end
+    alias_method :<<, :append
+
+    def append_with_dependents(*objects)
+      objects.each do |object|
+        object.dependencies.each { |dep| self.append dep }
+        self.append object
+      end
+      self
+    end
+  end
+end

data/lib/aw_datapipe/parameter_metadata.rb

@@ -0,0 +1,15 @@
+module AwDatapipe
+  class ParameterMetadata < Struct
+    def source
+      "{ " << members.map do |member|
+        member_source(member)
+      end.join(", ") << " }"
+    end
+
+    def member_source(member)
+      value = send member
+      value = ?' << value.gsub("'", "\\\\'") << ?' if value.is_a?(String)
+      "#{member}: #{value}"
+    end
+  end
+end

data/lib/aw_datapipe/pipeline.rb

@@ -0,0 +1,89 @@
+module AwDatapipe
+
+  class Pipeline
+    attr_accessor :id # AWS pipeline id
+    attr_accessor :uuid # Unique id
+    attr_reader :objects # ObjectHash[:id => PipelineObject]
+    attr_reader :parameter_metadata # Hash['parameterName' => Hash[type:|default:|description:|...]
+    attr_reader :parameter_values # Hash['parameterName' => "value"]
+
+    # objects [Array]
+    def initialize(objects, parameter_metadata, parameter_values)
+      objects.each { |object| object.pipeline = self }
+      @objects ||= ObjectHash.new(*objects)
+      @parameter_metadata, @parameter_values = parameter_metadata, parameter_values
+    end
+
+    def self.build(config, activities, parameter_metadata, parameter_values)
+      new([], parameter_metadata, parameter_values).tap { |p| p.objects.append_with_dependents(config, *activities) }
+    end
+
+    def configuration
+      objects.fetch(:default)
+    end
+
+    def referenced_object_ids
+      referenced_objects.map(&:id) << :default
+    end
+
+    # Collect dependencies for all objects, removing duplicates.
+    # @return [Array] referenced objects, with dependees before dependents.
+    def referenced_objects
+      objects.values.map(&:dependencies).flatten.uniq
+    end
+
+    def source
+      [
+        header_source,
+        objects_source,
+        parameter_metadata_source,
+        parameter_values_source,
+        footer_source
+      ].join("\n")
+    end
+
+    def write_source(pathname)
+      File.write(pathname, source)
+    end
+
+    protected
+
+    def footer_source
+      <<-EOF
+pipelines = AwDatapipe::Session.new
+pipeline = AwDatapipe::Pipeline.build(default, activities, parameter_metadata, parameter_values)
+pipeline.id = "#{id}"
+pipelines.save(pipeline)
+      EOF
+    end
+
+    def header_source
+      <<-EOF
+# Generated by aw_datapipe download_definition of #{id}
+require 'aw_datapipe'
+      EOF
+    end
+
+    def objects_source
+      object_ids = objects.keys
+      unreferenced_object_ids = object_ids - referenced_object_ids
+
+      s = referenced_object_ids.map { |id| "#{id} = #{objects[id].source}" }.join("\n\n")
+      s << "\n\nactivities = [\n"
+      s << unreferenced_object_ids.map { |id| "  #{objects[id].source(2)}" }.join(",\n")
+      s << "\n]"
+    end
+
+    def parameter_values_source
+      "parameter_values = {\n  " << parameter_values.sort.map do |key, value|
+        "\"#{key}\" => #{value.inspect}"
+      end.join(",\n  ") << "\n}\n"
+    end
+
+    def parameter_metadata_source
+      "parameter_metadata = {\n  " << parameter_metadata.sort.map do |key, value|
+        "\"#{key}\" => #{value.source}"
+      end.join(",\n  ") << "\n}\n"
+    end
+  end
+end

data/lib/aw_datapipe/pipeline_object.rb

@@ -0,0 +1,63 @@
+module AwDatapipe
+  class PipelineObject < Struct
+    attr_accessor :pipeline
+
+    def self.build(params)
+      new.tap do |struct|
+        params.each_pair { |k, v| struct.send "#{k}=", v }
+      end
+    end
+
+    # Iterates through struct members, recursively collecting any PipelineObjects.
+    # Recursion ensures dependencies sorted before dependents.
+    def dependencies
+      (members - [:id]).each_with_object([]) do |attr_name, depends|
+        value = send(attr_name)
+        value = pipeline.objects.fetch(value) if value.is_a?(Symbol)
+        depends << value.dependencies << value if value.is_a?(PipelineObject)
+      end.flatten
+    end
+
+    def demodulized_class_name
+      self.class.name.split('::').last
+    end
+
+    def to_hash
+      Hash[each_pair.to_a]
+    end
+
+    def source(indent_level = 1)
+      "#{self.class.name}.build(\n" << indent(indent_level) << members.map do |member|
+        member_source(member)
+      end.join(",\n" << indent(indent_level)) << ")"
+    end
+
+    protected
+
+    def indent(indent_level)
+      " " * 2 * indent_level
+    end
+
+    def member_source(member)
+      value = send member
+      value = ?' << value.gsub("'", "\\\\'") << ?' if value.is_a?(String)
+      value = ":#{value}" if member == :id
+      "#{member}: #{value}"
+    end
+  end
+
+  Configuration = PipelineObject.new(:failure_and_rerun_mode, :id, :name, :pipeline_log_uri, :resource_role, :role, :schedule, :schedule_type)
+  Schedule = PipelineObject.new(:id, :name, :period, :start_date_time)
+
+  Ec2Resource = PipelineObject.new(:action_on_task_failure, :id, :instance_type, :name, :security_group_ids, :subnet_id, :terminate_after)
+  S3DataNode = PipelineObject.new(:directory_path, :id, :name)
+
+  JdbcDatabase = PipelineObject.new(:_password, :connection_string, :id, :jdbc_driver_class, :name, :username)
+  SqlDataNode = PipelineObject.new(:database, :id, :name, :select_query, :table)
+  CopyActivity = PipelineObject.new(:id, :input, :name, :output, :runs_on)
+
+  RedshiftDatabase = PipelineObject.new(:_password, :connection_string, :database_name, :id, :name, :username)
+  RedshiftDataNode = PipelineObject.new(:create_table_sql, :database, :id, :name, :primary_keys, :schema_name, :table_name)
+  RedshiftCopyActivity = PipelineObject.new(:id, :input, :insert_mode, :name, :output, :runs_on)
+
+end

data/lib/aw_datapipe/pipeline_serializer.rb

@@ -0,0 +1,124 @@
+module AwDatapipe
+  # Converts a pipeline into a format that can be submitted to the AWS client SDK.
+  class PipelineSerializer
+    def marshal(pipeline)
+      {
+        pipeline_id: pipeline.id,
+        pipeline_objects: marshal_pipeline_objects(pipeline),
+        parameter_objects: marshal_parameter_objects(pipeline.parameter_metadata),
+        parameter_values: marshal_parameter_values(pipeline.parameter_values)
+      }
+    end
+
+    def unmarshal(aws_definition)
+      # pipeline.aws_definition = aws_definition # for troubleshooting
+      objects = unmarshal_pipeline_objects(aws_definition)
+      parameter_metadata = unmarshal_parameter_objects(aws_definition)
+      parameter_values = unmarshal_parameter_values(aws_definition)
+
+      Pipeline.new(objects, parameter_metadata, parameter_values)
+    end
+
+    protected
+
+    def marshal_pipeline_objects(pipeline)
+      # marshal referenced objects before unreferenced.
+      referenced_object_ids = pipeline.referenced_object_ids
+      unreferenced_object_ids = pipeline.objects.keys - referenced_object_ids
+      ids = (referenced_object_ids + unreferenced_object_ids)
+
+      ids.each_with_object([]) do |id, out|
+        out << marshal_pipeline_object(pipeline.objects[id])
+      end
+    end
+
+    def marshal_pipeline_object(pipeline_object)
+      type = pipeline_object.demodulized_class_name
+      hash = pipeline_object.to_hash
+      id = hash.delete(:id)
+      name = hash.delete(:name)
+      fields = hash_to_fields(hash)
+      fields << { key: 'type', string_value: type } unless type == 'Configuration'
+      Hash[id: camelize(id), name: name, fields: fields]
+    end
+
+    # @return Array PipelineObject subclass instance.
+    def unmarshal_pipeline_objects(aws_definition)
+      aws_definition.pipeline_objects.map do |aws_struct|
+        unmarshal_pipeline_object(aws_struct)
+      end
+    end
+
+    # @return PipelineObject subclass instance.
+    def unmarshal_pipeline_object(aws_struct)
+      attributes = fields_to_hash(aws_struct.fields)
+      type = attributes.delete(:type) || 'Configuration'
+      attributes.merge!(id: symbolize(aws_struct.id), name: aws_struct.name)
+
+      klass = AwDatapipe.const_defined?(type, false) ?
+        AwDatapipe.const_get(type, false) :
+        AwDatapipe.const_set(type, PipelineObject.new(*attributes.keys.sort))
+
+      klass.new(*attributes.sort.map(&:last)) # pass values sorted by keys
+    end
+
+    def fields_to_hash(fields)
+      fields.each_with_object({}) do |field, hash|
+        hash[symbolize field.key] = field.string_value || field.ref_value.underscore.to_sym
+      end
+    end
+
+    def hash_to_fields(hash)
+      hash.keys.map do |key|
+        PipelineObject === hash[key] ?
+          { key: camelize(key, :lower), ref_value: camelize(hash[key].id) } :
+          { key: camelize(key, :lower), string_value: hash[key] }
+      end
+    end
+
+    # Convert string to a rubyish variable name.
+    def symbolize(key)
+      key.underscore.gsub('*','_').to_sym
+    end
+
+    # Convert symbol back to AWSish name.
+    def camelize(key, term = :upper)
+      key.to_s.sub(/^\_/, '*').camelize(term)
+    end
+
+    def marshal_parameter_objects(parameter_metadata)
+      parameter_metadata.map { |key, hash| marshal_parameter_object(key, hash) }
+    end
+
+    def marshal_parameter_object(key, hash)
+      out = []
+      hash.each_pair do |k, v|
+        out << { key: k, string_value: v }
+      end
+      { id: key, attributes: out }
+    end
+
+    def unmarshal_parameter_objects(aws_definition)
+      aws_definition.parameter_objects.each_with_object({}) do |object, hash|
+        klass = ParameterMetadata.new(*object.attributes.map(&:key).map(&:to_sym))
+        hash[object.id] = object.attributes.each_with_object(klass.new) do |attribute, struct|
+          struct.send "#{attribute.key}=", attribute.string_value
+        end
+      end
+    end
+
+    def marshal_parameter_values(parameter_values)
+      out = []
+      parameter_values.each_pair do |id, value|
+        out << { id: id, string_value: value }
+      end
+      out
+    end
+
+    def unmarshal_parameter_values(aws_definition)
+      aws_definition.parameter_values.each_with_object({}) do |value, hash|
+        hash[value.id] = value.string_value
+      end
+    end
+  end
+end

data/lib/aw_datapipe/session.rb

@@ -0,0 +1,51 @@
+require 'securerandom'
+
+module AwDatapipe
+  class Session
+    def aws
+      @aws ||= Aws::DataPipeline::Client.new
+    end
+
+    def download_definition(id, dir)
+      fetch(id).write_source(dir)
+    end
+
+    # name [String] required
+    # description [String] (optional)
+    # unique_id [String] (default: uuid)
+    def create(name, description, unique_id = SecureRandom.uuid)
+      resp = aws.create_pipeline(name: name, unique_id: unique_id)
+      resp.pipeline_id
+    end
+
+    def keys
+      @keys ||= begin
+        resp = aws.list_pipelines
+        id_names = resp.pipeline_id_list
+        id_names.map(&:id)
+      end
+    end
+
+    def fetch(key)
+      resp = aws.get_pipeline_definition(pipeline_id: key)
+      serializer.unmarshal(resp).tap { |p| p.id = key }
+    end
+
+    def serializer
+      @serializer = PipelineSerializer.new
+    end
+
+    def save(pipeline)
+      resp = aws.put_pipeline_definition(serializer.marshal(pipeline))
+      if resp.errored
+        resp.validation_errors.each do |error|
+          puts "Error in #{error.id}: #{error.errors.inspect}"
+        end
+      end
+      resp.validation_warnings.each do |warning|
+        puts "Warning in #{warning.id}: #{warning.warnings.inspect}"
+      end
+      !resp.errored
+    end
+  end
+end

data/lib/aw_datapipe/version.rb

@@ -0,0 +1,3 @@
+module AwDatapipe
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: aw_datapipe
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Piers Chambers
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-05-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+description: Unofficial domain specific ruby wrapper for the AWS SDK Data Pipeline
+  API.
+email:
+- piers@varyonic.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- aw_datapipe.gemspec
+- bin/console
+- bin/setup
+- lib/aw_datapipe.rb
+- lib/aw_datapipe/object_hash.rb
+- lib/aw_datapipe/parameter_metadata.rb
+- lib/aw_datapipe/pipeline.rb
+- lib/aw_datapipe/pipeline_object.rb
+- lib/aw_datapipe/pipeline_serializer.rb
+- lib/aw_datapipe/session.rb
+- lib/aw_datapipe/version.rb
+homepage: http://github.com/varyonic/aw_datapipe
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: Unofficial ruby wrapper for the AWS SDK Data Pipeline API.
+test_files: []