pipe_fitter 0.1.8 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/exe/pipe_fitter +40 -0
- data/lib/pipe_fitter/data_pipeline_client.rb +64 -5
- data/lib/pipe_fitter/pipeline.rb +32 -3
- data/lib/pipe_fitter/version.rb +1 -1
- data/pipe_fitter.gemspec +1 -0
- metadata +17 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e48d74a4939c001d71118aa2beb5a6640cbfdee2
|
4
|
+
data.tar.gz: 22a9bfd2f2b2b84a5e66502238ab5463bec312a1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f724eaa70e4218f9f78b7529ed930b13c11cf1444db263d01142dccb2a6316af9fb2f722ca810b911c420771efc55f7fca30e23a5203209cfc2d81f19c494c64
|
7
|
+
data.tar.gz: 989b378d7811ce383b7d9e0719e3866c18745360563820632b001ee8189703a20d2c2be299de286704650237385ba01305b5c081fd9a210727c8f6e7fa2b5436
|
data/exe/pipe_fitter
CHANGED
@@ -49,6 +49,46 @@ module PipeFitter
|
|
49
49
|
puts Pipeline.load_yaml(definition_file).to_yaml
|
50
50
|
end
|
51
51
|
|
52
|
+
desc "find", "Find pipeline besed on name and uniqueId"
|
53
|
+
def find(definition_file)
|
54
|
+
puts client.find_registered(definition_file).to_h.to_json
|
55
|
+
end
|
56
|
+
|
57
|
+
desc "find_diff", "Find pipeline besed on name and uniqueId, show diff"
|
58
|
+
option :format, type: :string, default: "color", aliases: "f"
|
59
|
+
def find_diff(definition_file)
|
60
|
+
p = client.find_registered(definition_file)
|
61
|
+
abort("Pipeline is not registered") if p.nil?
|
62
|
+
puts client.diff(p.id, definition_file, options[:format])
|
63
|
+
end
|
64
|
+
|
65
|
+
desc "find_update", "Find pipeline besed on name and uniqueId, update pipeline definition"
|
66
|
+
option :force_update, type: :boolean, aliases: "f"
|
67
|
+
def find_update(definition_file)
|
68
|
+
p = client.find_registered(definition_file)
|
69
|
+
abort("Pipeline is not registered") if p.nil?
|
70
|
+
unless options[:force_update]
|
71
|
+
puts client.diff(p.id, definition_file)
|
72
|
+
print "\nReally update pipeline definition? [y/N] : "
|
73
|
+
abort("Update was canceled") if $stdin.gets.chomp !~ /^y$/i
|
74
|
+
end
|
75
|
+
res = client.update(p.id, definition_file)
|
76
|
+
puts JSON.pretty_generate(res)
|
77
|
+
end
|
78
|
+
|
79
|
+
desc "diff_deploy_files DEFINITION_FILE", "Show deploy files differences"
|
80
|
+
option :format, type: :string, default: "color", aliases: "f"
|
81
|
+
def diff_deploy_files(definition_file)
|
82
|
+
client.diff_deploy_files(definition_file, options[:format]).each do |d|
|
83
|
+
puts d
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
desc "upload_deploy_files DEFINITION_FILE", "Upload deploy files"
|
88
|
+
def upload_deploy_files(definition_file)
|
89
|
+
client.upload_deploy_files(definition_file)
|
90
|
+
end
|
91
|
+
|
52
92
|
private
|
53
93
|
|
54
94
|
def client
|
@@ -1,4 +1,6 @@
|
|
1
1
|
require "aws-sdk"
|
2
|
+
require "uri"
|
3
|
+
require "s3diff"
|
2
4
|
|
3
5
|
module PipeFitter
|
4
6
|
class NoSuchPipelineError < StandardError; end
|
@@ -9,17 +11,21 @@ module PipeFitter
|
|
9
11
|
end
|
10
12
|
|
11
13
|
def register(definition_file)
|
12
|
-
p =
|
14
|
+
p = load_pipeline(definition_file)
|
13
15
|
create(p)
|
14
16
|
end
|
15
17
|
|
16
18
|
def diff(pipeline_id, definition_file, format = :color)
|
17
|
-
p =
|
18
|
-
|
19
|
+
p = load_pipeline(definition_file)
|
20
|
+
[
|
21
|
+
definition(pipeline_id).diff(p, format.to_sym),
|
22
|
+
diff_deploy_files(definition_file, format.to_sym),
|
23
|
+
].compact.reject(&:empty?).join("\n")
|
19
24
|
end
|
20
25
|
|
21
26
|
def update(pipeline_id, definition_file)
|
22
|
-
|
27
|
+
upload_deploy_files(definition_file)
|
28
|
+
p = load_pipeline(definition_file)
|
23
29
|
put_definition(pipeline_id, p)
|
24
30
|
end
|
25
31
|
|
@@ -40,18 +46,57 @@ module PipeFitter
|
|
40
46
|
end
|
41
47
|
|
42
48
|
def activate(pipeline_id, parameter_file, start_timestamp)
|
43
|
-
p = parameter_file ?
|
49
|
+
p = parameter_file ? load_pipeline(parameter_file) : Pipeline.new
|
44
50
|
exec(:activate_pipeline, p.activate_opts(pipeline_id, start_timestamp)).to_h
|
45
51
|
end
|
46
52
|
|
53
|
+
def find_registered(definition_file)
|
54
|
+
p = load_pipeline(definition_file)
|
55
|
+
pls = list_pipelines.select { |l| l.name == p.pipeline_description.name }
|
56
|
+
res = pls.find do |pl|
|
57
|
+
d = Pipeline::PipelineDescription.create(description(pl.id))
|
58
|
+
d.unique_id == p.pipeline_description.unique_id
|
59
|
+
end
|
60
|
+
res
|
61
|
+
end
|
62
|
+
|
63
|
+
def diff_deploy_files(definition_file, format = :color)
|
64
|
+
p = load_pipeline(definition_file)
|
65
|
+
p.deploy_files.map do |df|
|
66
|
+
c = S3diff::Comparator.new(df[:dst], df[:src])
|
67
|
+
c.diff.to_s(format.to_sym) unless c.same?
|
68
|
+
end.compact
|
69
|
+
end
|
70
|
+
|
71
|
+
def upload_deploy_files(definition_file)
|
72
|
+
p = load_pipeline(definition_file)
|
73
|
+
p.deploy_files.each do |df|
|
74
|
+
put_object(df[:src], df[:dst])
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
47
78
|
private
|
48
79
|
|
80
|
+
def load_pipeline(definition_file)
|
81
|
+
Pipeline.load_yaml(definition_file)
|
82
|
+
end
|
83
|
+
|
49
84
|
def description(pipeline_id)
|
50
85
|
desc = exec(:describe_pipelines, pipeline_ids: [pipeline_id]).pipeline_description_list.first
|
51
86
|
raise NoSuchPipelineError, pipeline_id if desc.nil?
|
52
87
|
desc
|
53
88
|
end
|
54
89
|
|
90
|
+
def list_pipelines
|
91
|
+
res = exec(:list_pipelines)
|
92
|
+
pls = res.pipeline_id_list
|
93
|
+
while res.has_more_results
|
94
|
+
res = exec(:list_pipelines, marker: res.marker)
|
95
|
+
pls.concat(res.pipeline_id_list)
|
96
|
+
end
|
97
|
+
pls
|
98
|
+
end
|
99
|
+
|
55
100
|
def sync_tags(pipeline_id, pipeline)
|
56
101
|
p = definition(pipeline_id)
|
57
102
|
return if p.tags == pipeline.tags
|
@@ -65,10 +110,24 @@ module PipeFitter
|
|
65
110
|
raise NoSuchPipelineError, args.unshift(e.class)
|
66
111
|
end
|
67
112
|
|
113
|
+
def put_object(src, dst)
|
114
|
+
u = URI.parse(dst)
|
115
|
+
s3client.put_object(
|
116
|
+
body: File.read(src),
|
117
|
+
bucket: u.host,
|
118
|
+
key: u.path.sub(%r{^/}, "")
|
119
|
+
)
|
120
|
+
puts "put #{src} to #{dst}"
|
121
|
+
end
|
122
|
+
|
68
123
|
def client
|
69
124
|
@client ||= Aws::DataPipeline::Client.new(sdk_opts)
|
70
125
|
end
|
71
126
|
|
127
|
+
def s3client
|
128
|
+
@s3client ||= Aws::S3::Client.new(sdk_opts)
|
129
|
+
end
|
130
|
+
|
72
131
|
def sdk_opts
|
73
132
|
keys = %i(region profile).freeze
|
74
133
|
@options.select { |k, _| keys.include?(k) }
|
data/lib/pipe_fitter/pipeline.rb
CHANGED
@@ -7,6 +7,8 @@ module PipeFitter
|
|
7
7
|
class Pipeline
|
8
8
|
Diffy::Diff.default_options.merge!(diff: "-u", include_diff_info: true)
|
9
9
|
|
10
|
+
attr_reader :pipline_object, :parameter_objects, :parameter_values, :pipeline_description, :deploy_files
|
11
|
+
|
10
12
|
def self.create(definition_from_api, description_from_api)
|
11
13
|
new(PipelineObjects.create(definition_from_api[:pipeline_objects]),
|
12
14
|
ParameterObjects.create(definition_from_api[:parameter_objects]),
|
@@ -20,15 +22,17 @@ module PipeFitter
|
|
20
22
|
new(PipelineObjects.new(yml["pipeline_objects"]),
|
21
23
|
ParameterObjects.new(yml["parameter_objects"]),
|
22
24
|
ParameterValues.new(yml["parameter_values"]),
|
23
|
-
PipelineDescription.new(yml["pipeline_description"])
|
25
|
+
PipelineDescription.new(yml["pipeline_description"]),
|
26
|
+
DeployFiles.new(yml["deploy_files"], filepath))
|
24
27
|
end
|
25
28
|
|
26
29
|
def initialize(pipeline_objects = nil, parameter_objects = nil,
|
27
|
-
parameter_values = nil, pipeline_description = nil)
|
30
|
+
parameter_values = nil, pipeline_description = nil, deploy_files = nil)
|
28
31
|
@pipeline_objects = pipeline_objects
|
29
32
|
@parameter_objects = parameter_objects
|
30
33
|
@parameter_values = parameter_values
|
31
34
|
@pipeline_description = pipeline_description
|
35
|
+
@deploy_files = deploy_files
|
32
36
|
end
|
33
37
|
|
34
38
|
def tags
|
@@ -228,10 +232,11 @@ module PipeFitter
|
|
228
232
|
new(objs)
|
229
233
|
end
|
230
234
|
|
231
|
-
def initialize(objs)
|
235
|
+
def initialize(objs, filepath = nil)
|
232
236
|
@objs = symbolize_keys(objs || {}).sort_by do |k, v|
|
233
237
|
[DESCRIPTION_KEYS.index(k) || DESCRIPTION_KEYS.size + 1, k.to_s, v.to_s]
|
234
238
|
end.to_h
|
239
|
+
@filepath = filepath
|
235
240
|
end
|
236
241
|
|
237
242
|
DESCRIPTION_KEYS = %i(name description tags uniqueId).freeze
|
@@ -260,6 +265,30 @@ module PipeFitter
|
|
260
265
|
def tag_keys
|
261
266
|
@objs[:tags].map(&:keys).flatten
|
262
267
|
end
|
268
|
+
|
269
|
+
def name
|
270
|
+
@objs[:name]
|
271
|
+
end
|
272
|
+
|
273
|
+
def unique_id
|
274
|
+
@objs[:uniqueId]
|
275
|
+
end
|
276
|
+
end
|
277
|
+
|
278
|
+
class DeployFiles
|
279
|
+
include Enumerable
|
280
|
+
|
281
|
+
def initialize(objs, filepath)
|
282
|
+
@objs = objs
|
283
|
+
@filepath = filepath
|
284
|
+
end
|
285
|
+
|
286
|
+
def each
|
287
|
+
(@objs || []).map do |df|
|
288
|
+
h = { src: File.join(@filepath.dirname, df["src"]), dst: df["dst"] }
|
289
|
+
yield h
|
290
|
+
end
|
291
|
+
end
|
263
292
|
end
|
264
293
|
end
|
265
294
|
end
|
data/lib/pipe_fitter/version.rb
CHANGED
data/pipe_fitter.gemspec
CHANGED
@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
|
|
23
23
|
spec.add_runtime_dependency "diffy"
|
24
24
|
spec.add_runtime_dependency "thor"
|
25
25
|
spec.add_runtime_dependency "hashie"
|
26
|
+
spec.add_runtime_dependency "s3diff"
|
26
27
|
|
27
28
|
spec.add_development_dependency "bundler", "~> 1.11"
|
28
29
|
spec.add_development_dependency "rake", "~> 10.0"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pipe_fitter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- masa21kik
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-09-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk
|
@@ -66,6 +66,20 @@ dependencies:
|
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: s3diff
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
69
83
|
- !ruby/object:Gem::Dependency
|
70
84
|
name: bundler
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -193,7 +207,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
193
207
|
version: '0'
|
194
208
|
requirements: []
|
195
209
|
rubyforge_project:
|
196
|
-
rubygems_version: 2.
|
210
|
+
rubygems_version: 2.6.13
|
197
211
|
signing_key:
|
198
212
|
specification_version: 4
|
199
213
|
summary: PipeFitter is a tool for AWS Data Pipeline.
|