pipe_fitter 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8b23b1fe9578e816dd779dc1a5cb44995d1290c1
+  data.tar.gz: b5035699203388bfc31654839cf84d2d44cb3644
+SHA512:
+  metadata.gz: bfb1784675d8dfa25567adcc5ace6f5828387c2aaef3d107daa5d7055978bf19332ab2adbf128b42f9d9871b069c10c7f12a1abc4e6e0ed878e76b445cbfa546
+  data.tar.gz: 44fa120401769db6b94bf3b59c5ef1ca0c537462aa9cf2ecd0f65d76cd55d9df1c84a669229de78da7fc68716cf4683bd548692a73ea83580ec9bf7331f83a15

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/vendor/

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.3.0
+before_install: gem install bundler -v 1.11.2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in pipe_fitter.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 masa21kik
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,49 @@
+# PipeFitter
+
+PipeFitter is a tool for AWS Data Pipeline.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'pipe_fitter'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install pipe_fitter
+
+## Usage
+
+Register pipeline.
+
+```
+pipe_fitter register <pipeline name>
+```
+
+Show difference.
+
+```
+pipe_fitter diff <pipeline name>
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/pipe_fitter.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+desc "Run test_unit based test"
+Rake::TestTask.new do |t|
+  # To run test for only one file (or file path pattern)
+  #  $ bundle exec rake test TEST=test/test_specified_path.rb
+  t.libs << "test"
+  t.test_files = Dir["test/**/test_*.rb"]
+  t.verbose = true
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "pipe_fitter"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+require "pry"
+Pry.start
+
+# require "irb"
+# IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/pipe_filter

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require "pipe_fitter/cli"
+PipeFitter::Cli.start

data/lib/pipe_fitter.rb

@@ -0,0 +1,7 @@
+require "pipe_fitter/version"
+require "pipe_fitter/pipeline"
+require "pipe_fitter/yaml_loader"
+require "pipe_fitter/data_pipeline_client"
+
+module PipeFitter
+end

data/lib/pipe_fitter/cli.rb

@@ -0,0 +1,50 @@
+require "pipe_fitter"
+require "thor"
+
+module PipeFitter
+  class Cli < Thor
+    class_option :region, type: :string
+    class_option :profile, type: :string
+
+    desc "export", "Export pipeline definition"
+    option :pipeline_id, type: :string, required: true, aliases: "i"
+    def export
+      puts client.definition(options[:pipeline_id]).to_yaml
+    end
+
+    desc "register DEFINITION_FILE", "Register pipeline"
+    def register(definition_file)
+      id, res = client.register(definition_file)
+      puts id, JSON.pretty_generate(res)
+    end
+
+    desc "diff DEFINITION_FILE", "Show pipeline difference"
+    option :pipeline_id, type: :string, required: true, aliases: "i"
+    option :format, type: :string, default: "color", aliases: "f"
+    def diff(definition_file)
+      puts client.diff(options[:pipeline_id], definition_file, options[:format])
+    end
+
+    desc "update DEFINITION_FILE", "Update pipeline definition"
+    option :pipeline_id, type: :string, required: true, aliases: "i"
+    def update(definition_file)
+      res = client.update(options[:pipeline_id], definition_file)
+      puts JSON.pretty_generate(res)
+    end
+
+    desc "activate pipeline", "Activate pipeline"
+    option :pipeline_id, type: :string, required: true, aliases: "i"
+    option :parameter_values, type: :hash, required: false, aliases: "p"
+    option :start_timestamp, type: :string, required: false, aliases: "t"
+    def activate
+      t = options[:start_timestamp] ? Time.parse(options[:start_timestamp]) : nil
+      puts client.activate(options[:pipeline_id], options[:parameter_values], t)
+    end
+
+    private
+
+    def client
+      @client ||= DataPipelineClient.new(options)
+    end
+  end
+end

data/lib/pipe_fitter/data_pipeline_client.rb

@@ -0,0 +1,77 @@
+require "aws-sdk"
+
+module PipeFitter
+  class NoSuchPipelineError < StandardError; end
+
+  class DataPipelineClient
+    def initialize(options)
+      @options = options.map { |k, v| [k.to_sym, v] }.to_h
+    end
+
+    def register(definition_file)
+      p = Pipeline.load_yaml(definition_file)
+      create(p)
+    end
+
+    def diff(pipeline_id, definition_file, format = :color)
+      p = Pipeline.load_yaml(definition_file)
+      definition(pipeline_id).diff(p, format.to_sym)
+    end
+
+    def update(pipeline_id, definition_file)
+      p = Pipeline.load_yaml(definition_file)
+      put_definition(pipeline_id, p)
+    end
+
+    def definition(pipeline_id)
+      res = exec(:get_pipeline_definition, pipeline_id: pipeline_id)
+      desc = description(pipeline_id)
+      Pipeline.new(res.to_h, desc.to_h)
+    end
+
+    def put_definition(pipeline_id, pipeline)
+      sync_tags(pipeline_id, pipeline)
+      exec(:put_pipeline_definition, pipeline.put_definition_opts(pipeline_id)).to_h
+    end
+
+    def create(pipeline)
+      res = exec(:create_pipeline, pipeline.create_opts)
+      [res.pipeline_id, put_definition(res.pipeline_id, pipeline)]
+    end
+
+    def activate(pipeline_id, parameter_values, start_timestamp)
+      p = Pipeline.new({}, { pipeline_id: pipeline_id })
+      exec(:activate_pipeline, p.activate_opts(parameter_values, start_timestamp))
+    end
+
+    private
+
+    def description(pipeline_id)
+      desc = exec(:describe_pipelines, pipeline_ids: [pipeline_id]).pipeline_description_list.first
+      raise NoSuchPipelineError, pipeline_id if desc.nil?
+      desc
+    end
+
+    def sync_tags(pipeline_id, pipeline)
+      p = definition(pipeline_id)
+      return if p.tags == pipeline.tags
+      exec(:remove_tags, p.remove_tags_opts) unless p.tags.empty?
+      exec(:add_tags, pipeline.add_tags_opts(pipeline_id)) unless pipeline.tags.empty?
+    end
+
+    def exec(method, *args)
+      client.send(method, *args)
+    rescue Aws::DataPipeline::Errors::PipelineNotFoundException, Aws::DataPipeline::Errors::PipelineDeletedException => e
+      raise NoSuchPipelineError, args.unshift(e.class)
+    end
+
+    def client
+      @client ||= Aws::DataPipeline::Client.new(sdk_opts)
+    end
+
+    def sdk_opts
+      keys = %i(region profile).freeze
+      @options.select { |k, _| keys.include?(k) }
+    end
+  end
+end

data/lib/pipe_fitter/pipeline.rb

@@ -0,0 +1,112 @@
+require "yaml"
+require "hashie"
+require "diffy"
+require "pathname"
+
+module PipeFitter
+  class Pipeline
+    class << self
+      def load_yaml(filename)
+        filepath = Pathname.new(filename)
+        desc = YamlLoader.load(filepath)
+        definition = desc.delete("definition")
+        Pipeline.new(definition, desc)
+      end
+    end
+
+    DESCRIPTION_KEYS = %w(name tags unique_id description).freeze
+    Diffy::Diff.default_options.merge!(diff: "-u", include_diff_info: true)
+
+    def initialize(definition, description = {})
+      @definition = Hashie::Mash.new(definition)
+      @full_description = Hashie::Mash.new(description).tap do |h|
+        if (fs = h[:fields]) && (f = fs.find { |e| e[:key] == "uniqueId" }) && f.key?(:string_value)
+          h.unique_id ||= f[:string_value]
+        end
+      end
+      @description = Hashie::Mash.new(@full_description.select { |k, v| DESCRIPTION_KEYS.include?(k) && !v.nil? })
+    end
+
+    def to_yaml
+      stringify_keys(@description.merge("definition" => sorted_definition)).to_yaml
+    end
+
+    def create_opts
+      symbolize_keys(@description)
+    end
+
+    def put_definition_opts(id = nil)
+      base = { pipeline_id: id || pipeline_id }
+      base.merge(symbolize_keys(@definition))
+    end
+
+    def add_tags_opts(id = nil)
+      { pipeline_id: id || pipeline_id, tags: @full_description.tags }
+    end
+
+    def remove_tags_opts(id = nil)
+      { pipeline_id: id || pipeline_id, tag_keys: @full_description.tags.map { |e| e["key"] } }
+    end
+
+    def activate_opts(parameter_values, start_timestamp = nil, id = nil)
+      pv = (parameter_values || {}).map { |k, v| { "id" => k.to_s, "string_value" => v.to_s } }
+      { pipeline_id: id || pipeline_id, parameter_values: pv, start_timestamp: start_timestamp }.select { |_, v| !v.nil? }
+    end
+
+    def pipeline_id
+      @full_description.pipeline_id
+    end
+
+    def name
+      @full_description.name
+    end
+
+    def tags
+      @full_description.tags
+    end
+
+    def diff(other, format = nil)
+      Diffy::Diff.new(self.to_yaml, other.to_yaml).to_s(format)
+    end
+
+    private
+
+    def sorted_definition
+      top_keys = %w(pipeline_objects parameter_objects parameter_values).freeze
+      obj_keys = %w(id name fields).freeze
+      field_keys = %w(key string_value ref_value).freeze
+
+      d = @definition.sort_by { |k, _| [top_keys.index(k) || top_keys.size, k] }.to_h
+      top_keys.each do |k|
+        next if !d[k].is_a?(Array) || d[k].empty?
+        d[k] = d[k].sort_by { |e| e["id"] }
+        d[k].map! do |v|
+          v["fields"] = v["fields"].sort_by { |e| e["key"] }
+          v["fields"].map! { |vv| vv.sort_by { |kk, _| field_keys.index(kk) }.to_h }
+          v.sort_by { |kk, _| obj_keys.index(kk) }.to_h
+        end
+      end
+      d
+    end
+
+    def stringify_keys(val)
+      modify_keys_recursively(val, __method__)
+    end
+
+    def symbolize_keys(val)
+      modify_keys_recursively(val, __method__)
+    end
+
+    def modify_keys_recursively(val, method)
+      return val unless val.is_a?(Hash)
+      h = Hashie.send(method, val.to_h)
+      h.each do |k, v|
+        case v
+        when Array then h[k].map! { |e| self.send(method, e) }
+        when Hash then h[k] = self.send(method, v)
+        end
+      end
+      h
+    end
+  end
+end

data/lib/pipe_fitter/version.rb

@@ -0,0 +1,3 @@
+module PipeFitter
+  VERSION = "0.1.0"
+end

data/lib/pipe_fitter/yaml_loader.rb

@@ -0,0 +1,24 @@
+require "yaml"
+require "erb"
+
+module PipeFitter
+  class YamlLoader
+    class << self
+      def load(filename, context_filename = nil)
+        context = context_filename.nil? ? nil : YAML.load_file(context_filename)
+        YAML.load(eval_erb(filename, context)) || {}
+      end
+
+      def include_yaml(filepath, indent = 1)
+        load(filepath).to_yaml.chomp.gsub("---", "").gsub("\n", "\n" + "  " * indent)
+      end
+
+      private
+
+      def eval_erb(filename, context = nil)
+        context ||= {}
+        ERB.new(File.read(filename)).result(binding)
+      end
+    end
+  end
+end

data/pipe_fitter.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'pipe_fitter/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "pipe_fitter"
+  spec.version       = PipeFitter::VERSION
+  spec.authors       = ["masa21kik"]
+  spec.email         = ["masa21kik@gmail.com"]
+
+  spec.summary       = %q{PipeFitter is a tool for AWS Data Pipeline.}
+  spec.description   = %q{PipeFitter is a tool for AWS Data Pipeline.}
+  spec.homepage      = "https://github.com/masa21kik/pipe_fitter"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "aws-sdk", "~> 2"
+  spec.add_runtime_dependency "diffy"
+  spec.add_runtime_dependency "thor"
+  spec.add_runtime_dependency "hashie"
+
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "test-unit"
+  spec.add_development_dependency "test-unit-rr"
+  spec.add_development_dependency "pry-byebug"
+end

metadata

@@ -0,0 +1,201 @@
+--- !ruby/object:Gem::Specification
+name: pipe_fitter
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- masa21kik
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-06-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: diffy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: PipeFitter is a tool for AWS Data Pipeline.
+email:
+- masa21kik@gmail.com
+executables:
+- pipe_filter
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/pipe_filter
+- lib/pipe_fitter.rb
+- lib/pipe_fitter/cli.rb
+- lib/pipe_fitter/data_pipeline_client.rb
+- lib/pipe_fitter/pipeline.rb
+- lib/pipe_fitter/version.rb
+- lib/pipe_fitter/yaml_loader.rb
+- pipe_fitter.gemspec
+homepage: https://github.com/masa21kik/pipe_fitter
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: PipeFitter is a tool for AWS Data Pipeline.
+test_files: []