pipe_fitter 0.1.8 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 892c113ee708c1620229690aa80940b4cd2ec4df
-  data.tar.gz: ec8b73ebbd4f064f944ee640158782d76291b16d
+  metadata.gz: e48d74a4939c001d71118aa2beb5a6640cbfdee2
+  data.tar.gz: 22a9bfd2f2b2b84a5e66502238ab5463bec312a1
 SHA512:
-  metadata.gz: 5a3801e39e26c40dd1dda5411d23fce072f3d95f56496b600dfc02a98f4ed29b0dc8f6732c8469bd6f089ec69c75fafc07eafb7e6c89af764cf08c8325ad4679
-  data.tar.gz: 8053e25626a894251dd2bb93b7c261728c64bac31fc6984d3847713114a02a11da0cc79732d02dc7d038718854c835349b1cf6af9bb1608644181aff33283bf9
+  metadata.gz: f724eaa70e4218f9f78b7529ed930b13c11cf1444db263d01142dccb2a6316af9fb2f722ca810b911c420771efc55f7fca30e23a5203209cfc2d81f19c494c64
+  data.tar.gz: 989b378d7811ce383b7d9e0719e3866c18745360563820632b001ee8189703a20d2c2be299de286704650237385ba01305b5c081fd9a210727c8f6e7fa2b5436

data/exe/pipe_fitter

@@ -49,6 +49,46 @@ module PipeFitter
       puts Pipeline.load_yaml(definition_file).to_yaml
     end
 
+    desc "find", "Find pipeline besed on name and uniqueId"
+    def find(definition_file)
+      puts client.find_registered(definition_file).to_h.to_json
+    end
+
+    desc "find_diff", "Find pipeline besed on name and uniqueId, show diff"
+    option :format, type: :string, default: "color", aliases: "f"
+    def find_diff(definition_file)
+      p = client.find_registered(definition_file)
+      abort("Pipeline is not registered") if p.nil?
+      puts client.diff(p.id, definition_file, options[:format])
+    end
+
+    desc "find_update", "Find pipeline besed on name and uniqueId, update pipeline definition"
+    option :force_update, type: :boolean, aliases: "f"
+    def find_update(definition_file)
+      p = client.find_registered(definition_file)
+      abort("Pipeline is not registered") if p.nil?
+      unless options[:force_update]
+        puts client.diff(p.id, definition_file)
+        print "\nReally update pipeline definition? [y/N] : "
+        abort("Update was canceled") if $stdin.gets.chomp !~ /^y$/i
+      end
+      res = client.update(p.id, definition_file)
+      puts JSON.pretty_generate(res)
+    end
+
+    desc "diff_deploy_files DEFINITION_FILE", "Show deploy files differences"
+    option :format, type: :string, default: "color", aliases: "f"
+    def diff_deploy_files(definition_file)
+      client.diff_deploy_files(definition_file, options[:format]).each do |d|
+        puts d
+      end
+    end
+
+    desc "upload_deploy_files DEFINITION_FILE", "Upload deploy files"
+    def upload_deploy_files(definition_file)
+      client.upload_deploy_files(definition_file)
+    end
+
     private
 
     def client

data/lib/pipe_fitter/data_pipeline_client.rb

@@ -1,4 +1,6 @@
 require "aws-sdk"
+require "uri"
+require "s3diff"
 
 module PipeFitter
   class NoSuchPipelineError < StandardError; end
@@ -9,17 +11,21 @@ module PipeFitter
     end
 
     def register(definition_file)
-      p = Pipeline.load_yaml(definition_file)
+      p = load_pipeline(definition_file)
       create(p)
     end
 
     def diff(pipeline_id, definition_file, format = :color)
-      p = Pipeline.load_yaml(definition_file)
-      definition(pipeline_id).diff(p, format.to_sym)
+      p = load_pipeline(definition_file)
+      [
+        definition(pipeline_id).diff(p, format.to_sym),
+        diff_deploy_files(definition_file, format.to_sym),
+      ].compact.reject(&:empty?).join("\n")
     end
 
     def update(pipeline_id, definition_file)
-      p = Pipeline.load_yaml(definition_file)
+      upload_deploy_files(definition_file)
+      p = load_pipeline(definition_file)
       put_definition(pipeline_id, p)
     end
 
@@ -40,18 +46,57 @@ module PipeFitter
     end
 
     def activate(pipeline_id, parameter_file, start_timestamp)
-      p = parameter_file ? Pipeline.load_yaml(parameter_file) : Pipeline.new
+      p = parameter_file ? load_pipeline(parameter_file) : Pipeline.new
       exec(:activate_pipeline, p.activate_opts(pipeline_id, start_timestamp)).to_h
     end
 
+    def find_registered(definition_file)
+      p = load_pipeline(definition_file)
+      pls = list_pipelines.select { |l| l.name == p.pipeline_description.name }
+      res = pls.find do |pl|
+        d = Pipeline::PipelineDescription.create(description(pl.id))
+        d.unique_id == p.pipeline_description.unique_id
+      end
+      res
+    end
+
+    def diff_deploy_files(definition_file, format = :color)
+      p = load_pipeline(definition_file)
+      p.deploy_files.map do |df|
+        c = S3diff::Comparator.new(df[:dst], df[:src])
+        c.diff.to_s(format.to_sym) unless c.same?
+      end.compact
+    end
+
+    def upload_deploy_files(definition_file)
+      p = load_pipeline(definition_file)
+      p.deploy_files.each do |df|
+        put_object(df[:src], df[:dst])
+      end
+    end
+
     private
 
+    def load_pipeline(definition_file)
+      Pipeline.load_yaml(definition_file)
+    end
+
     def description(pipeline_id)
       desc = exec(:describe_pipelines, pipeline_ids: [pipeline_id]).pipeline_description_list.first
       raise NoSuchPipelineError, pipeline_id if desc.nil?
       desc
     end
 
+    def list_pipelines
+      res = exec(:list_pipelines)
+      pls = res.pipeline_id_list
+      while res.has_more_results
+        res = exec(:list_pipelines, marker: res.marker)
+        pls.concat(res.pipeline_id_list)
+      end
+      pls
+    end
+
     def sync_tags(pipeline_id, pipeline)
       p = definition(pipeline_id)
       return if p.tags == pipeline.tags
@@ -65,10 +110,24 @@ module PipeFitter
       raise NoSuchPipelineError, args.unshift(e.class)
     end
 
+    def put_object(src, dst)
+      u = URI.parse(dst)
+      s3client.put_object(
+        body: File.read(src),
+        bucket: u.host,
+        key: u.path.sub(%r{^/}, "")
+      )
+      puts "put #{src} to #{dst}"
+    end
+
     def client
       @client ||= Aws::DataPipeline::Client.new(sdk_opts)
     end
 
+    def s3client
+      @s3client ||= Aws::S3::Client.new(sdk_opts)
+    end
+
     def sdk_opts
       keys = %i(region profile).freeze
       @options.select { |k, _| keys.include?(k) }

data/lib/pipe_fitter/pipeline.rb

@@ -7,6 +7,8 @@ module PipeFitter
   class Pipeline
     Diffy::Diff.default_options.merge!(diff: "-u", include_diff_info: true)
 
+    attr_reader :pipline_object, :parameter_objects, :parameter_values, :pipeline_description, :deploy_files
+
     def self.create(definition_from_api, description_from_api)
       new(PipelineObjects.create(definition_from_api[:pipeline_objects]),
           ParameterObjects.create(definition_from_api[:parameter_objects]),
@@ -20,15 +22,17 @@ module PipeFitter
       new(PipelineObjects.new(yml["pipeline_objects"]),
           ParameterObjects.new(yml["parameter_objects"]),
           ParameterValues.new(yml["parameter_values"]),
-          PipelineDescription.new(yml["pipeline_description"]))
+          PipelineDescription.new(yml["pipeline_description"]),
+          DeployFiles.new(yml["deploy_files"], filepath))
     end
 
     def initialize(pipeline_objects = nil, parameter_objects = nil,
-                   parameter_values = nil, pipeline_description = nil)
+                   parameter_values = nil, pipeline_description = nil, deploy_files = nil)
       @pipeline_objects = pipeline_objects
       @parameter_objects = parameter_objects
       @parameter_values = parameter_values
       @pipeline_description = pipeline_description
+      @deploy_files = deploy_files
     end
 
     def tags
@@ -228,10 +232,11 @@ module PipeFitter
         new(objs)
       end
 
-      def initialize(objs)
+      def initialize(objs, filepath = nil)
         @objs = symbolize_keys(objs || {}).sort_by do |k, v|
           [DESCRIPTION_KEYS.index(k) || DESCRIPTION_KEYS.size + 1, k.to_s, v.to_s]
         end.to_h
+        @filepath = filepath
       end
 
       DESCRIPTION_KEYS = %i(name description tags uniqueId).freeze
@@ -260,6 +265,30 @@ module PipeFitter
       def tag_keys
         @objs[:tags].map(&:keys).flatten
       end
+
+      def name
+        @objs[:name]
+      end
+
+      def unique_id
+        @objs[:uniqueId]
+      end
+    end
+
+    class DeployFiles
+      include Enumerable
+
+      def initialize(objs, filepath)
+        @objs = objs
+        @filepath = filepath
+      end
+
+      def each
+        (@objs || []).map do |df|
+          h = { src: File.join(@filepath.dirname, df["src"]), dst: df["dst"] }
+          yield h
+        end
+      end
     end
   end
 end

data/lib/pipe_fitter/version.rb

@@ -1,3 +1,3 @@
 module PipeFitter
-  VERSION = "0.1.8"
+  VERSION = "0.2.0"
 end

data/pipe_fitter.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "diffy"
   spec.add_runtime_dependency "thor"
   spec.add_runtime_dependency "hashie"
+  spec.add_runtime_dependency "s3diff"
 
   spec.add_development_dependency "bundler", "~> 1.11"
   spec.add_development_dependency "rake", "~> 10.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pipe_fitter
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.2.0
 platform: ruby
 authors:
 - masa21kik
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-06-19 00:00:00.000000000 Z
+date: 2017-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: s3diff
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -193,7 +207,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: PipeFitter is a tool for AWS Data Pipeline.