patriot-aws 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -13
- data/lib/patriot_aws/command.rb +1 -0
- data/lib/patriot_aws/command/redshift.rb +121 -0
- data/lib/patriot_aws/command/s3.rb +0 -0
- data/lib/patriot_aws/version.rb +1 -1
- metadata +24 -10
checksums.yaml
CHANGED
@@ -1,15 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
|
5
|
-
data.tar.gz: !binary |-
|
6
|
-
ZGMwYmJhNDkwZGIwYzhlYjdjZjM2NTQwZjNlNDljNmUyMDBmMzVlYg==
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 26b16fcbab39dffb44c42a4badeb96236a388835
|
4
|
+
data.tar.gz: 7dcafac13fbbcfd9b701dc296697e99669058edd
|
7
5
|
SHA512:
|
8
|
-
metadata.gz:
|
9
|
-
|
10
|
-
ODBlZmZjMDM1OWIxNDQxZjhlOGM0YzEyM2IzZWJiOGE1YjRmNTdhMzA2OGEx
|
11
|
-
NjQzZTdiMWI1YWYzM2EyODcwODg1ZTdlNWZkNGM4ZGMyNzJhNWE=
|
12
|
-
data.tar.gz: !binary |-
|
13
|
-
ODZmNTI4OWY4OWM1MzdjMDI0YzM0MWRhYjhmYWEwMmM4MGFhMjc5NTllMWRm
|
14
|
-
ODRhOTE2Zjg4NDNiYjY1NDY1NTg0ZTMwMWU2OTMyYmU1N2QxMTMzNjBkOGIz
|
15
|
-
OTkyOTU4ODgxMjBhNWNkMGM5OWY1MDhjZDAzZTM2OTkyZjRkMjE=
|
6
|
+
metadata.gz: 780e3b20dfa8cb4aec12935a29bbf378df59639ab5c5f981e7e343adce9ddcb2421c0bdcca2facf051e8912244b20b749329e12a9e859d52a22208c6f2462376
|
7
|
+
data.tar.gz: 450bd34d82813b575a2b66acff47d20c3fe3e7bc3bbd3e1bb19f11ccb88acb1b135d562c234396ce7355da2236d6c8687780f8eeea34346f30b30dc0bac99184
|
data/lib/patriot_aws/command.rb
CHANGED
@@ -0,0 +1,121 @@
|
|
1
|
+
=begin
|
2
|
+
|
3
|
+
sample_select.pbc
|
4
|
+
--
|
5
|
+
redshift {
|
6
|
+
name "select_from_redshift"
|
7
|
+
name_suffix _date_
|
8
|
+
inifile '/path/to//redshift.ini'
|
9
|
+
query 'select * from test_table'
|
10
|
+
options :with_header => true, :delimiter => "\t"
|
11
|
+
}
|
12
|
+
|
13
|
+
sample_copy.pbc
|
14
|
+
--
|
15
|
+
redshift {
|
16
|
+
name "s3_to_redshift"
|
17
|
+
name_suffix _date_
|
18
|
+
inifile '/Users/b06959/tmp/redshift.ini'
|
19
|
+
query <<-EOS
|
20
|
+
COPY #{schema}.#{table}
|
21
|
+
FROM '#{s3_path}'
|
22
|
+
ACCESS_KEY_ID '%{access_key_id}'
|
23
|
+
SECRET_ACCESS_KEY '%{secret_access_key}'
|
24
|
+
delimiter '\t'
|
25
|
+
gzip
|
26
|
+
EOS
|
27
|
+
}
|
28
|
+
|
29
|
+
$ cat redshift.ini
|
30
|
+
[connection]
|
31
|
+
host = staging.xxxxxxxxxxxxx.ap-northeast-1.redshift.amazonaws.com
|
32
|
+
user = staging
|
33
|
+
password = staging
|
34
|
+
dbname = staging
|
35
|
+
port = 5439
|
36
|
+
|
37
|
+
[s3credentials]
|
38
|
+
access_key_id = #{access_key_id},
|
39
|
+
secret_access_key = #{secret_access_key}
|
40
|
+
=end
|
41
|
+
|
42
|
+
require 'pg'
|
43
|
+
|
44
|
+
module PatriotAWS
|
45
|
+
module Command
|
46
|
+
class RedshiftCommand < Patriot::Command::Base
|
47
|
+
declare_command_name :redshift
|
48
|
+
include PatriotAWS::Ext::AWS
|
49
|
+
|
50
|
+
command_attr :name, :name_suffix, :inifile, :options, :query
|
51
|
+
|
52
|
+
def job_id
|
53
|
+
job_id = "#{command_name}_#{@name}_#{@name_suffix}"
|
54
|
+
job_id
|
55
|
+
end
|
56
|
+
|
57
|
+
# @see Patriot::Command::Base#configure
|
58
|
+
def configure
|
59
|
+
@name_suffix ||= _date_
|
60
|
+
self
|
61
|
+
end
|
62
|
+
|
63
|
+
def execute
|
64
|
+
@logger.info 'start redshift query...'
|
65
|
+
@options ||= {}
|
66
|
+
|
67
|
+
ini = IniFile.load(@inifile)
|
68
|
+
raise Exception, 'inifile not found.' if ini.nil?
|
69
|
+
raise Exception, 'query is not set.' if @query.nil?
|
70
|
+
|
71
|
+
_set_options
|
72
|
+
|
73
|
+
begin
|
74
|
+
# replace variables
|
75
|
+
if ini['s3credentials']
|
76
|
+
@query = @query % ini['s3credentials'].symbolize_keys
|
77
|
+
end
|
78
|
+
|
79
|
+
conn = PG::Connection.new(
|
80
|
+
ini['connection'].symbolize_keys
|
81
|
+
)
|
82
|
+
res = conn.exec(@query)
|
83
|
+
|
84
|
+
output_arr = Array.new
|
85
|
+
if res then
|
86
|
+
res.each_with_index do |r, idx|
|
87
|
+
# print header
|
88
|
+
if @options[:with_header] && idx == 0
|
89
|
+
output_arr.push r.keys.join(@options[:delimiter])
|
90
|
+
end
|
91
|
+
|
92
|
+
output_arr.push r.values.join(@options[:delimiter])
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
puts output_arr.join("\n")
|
97
|
+
# rescue PGError => ex
|
98
|
+
# # may cause PGError when connection setting is invalid
|
99
|
+
# # e.g.
|
100
|
+
# # PG::ConnectionBad -> could not translate host name "test" to address: nodename nor servname provided, or not known
|
101
|
+
# print(ex.class," -> ",ex.message)
|
102
|
+
#
|
103
|
+
# raise PGError
|
104
|
+
# rescue => ex
|
105
|
+
# # Other Error process
|
106
|
+
# print(ex.class," -> ",ex.message)
|
107
|
+
ensure
|
108
|
+
conn.close if conn
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
# @private
|
113
|
+
# set default option parameters
|
114
|
+
def _set_options
|
115
|
+
@options[:with_header] = false if @options[:with_header].nil?
|
116
|
+
@options[:delimiter] = "\t" if @options[:delimiter].nil?
|
117
|
+
end
|
118
|
+
private :_set_options
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
File without changes
|
data/lib/patriot_aws/version.rb
CHANGED
metadata
CHANGED
@@ -1,41 +1,55 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: patriot-aws
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Takayuki Tanaka
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-06-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - ~>
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '2'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - ~>
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '2'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: pg
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.18.4
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 0.18.4
|
27
41
|
- !ruby/object:Gem::Dependency
|
28
42
|
name: patriot-workflow-scheduler
|
29
43
|
requirement: !ruby/object:Gem::Requirement
|
30
44
|
requirements:
|
31
|
-
- - ~>
|
45
|
+
- - "~>"
|
32
46
|
- !ruby/object:Gem::Version
|
33
47
|
version: '0.7'
|
34
48
|
type: :runtime
|
35
49
|
prerelease: false
|
36
50
|
version_requirements: !ruby/object:Gem::Requirement
|
37
51
|
requirements:
|
38
|
-
- - ~>
|
52
|
+
- - "~>"
|
39
53
|
- !ruby/object:Gem::Version
|
40
54
|
version: '0.7'
|
41
55
|
description: plugins for Patriot Workflow Scheduler, which deal with AWS such as S3.
|
@@ -48,6 +62,7 @@ files:
|
|
48
62
|
- init.rb
|
49
63
|
- lib/patriot_aws.rb
|
50
64
|
- lib/patriot_aws/command.rb
|
65
|
+
- lib/patriot_aws/command/redshift.rb
|
51
66
|
- lib/patriot_aws/command/s3.rb
|
52
67
|
- lib/patriot_aws/ext.rb
|
53
68
|
- lib/patriot_aws/ext/aws.rb
|
@@ -62,19 +77,18 @@ require_paths:
|
|
62
77
|
- lib
|
63
78
|
required_ruby_version: !ruby/object:Gem::Requirement
|
64
79
|
requirements:
|
65
|
-
- -
|
80
|
+
- - ">="
|
66
81
|
- !ruby/object:Gem::Version
|
67
82
|
version: '0'
|
68
83
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
69
84
|
requirements:
|
70
|
-
- -
|
85
|
+
- - ">="
|
71
86
|
- !ruby/object:Gem::Version
|
72
87
|
version: '0'
|
73
88
|
requirements: []
|
74
89
|
rubyforge_project: patriot-aws
|
75
|
-
rubygems_version: 2.
|
90
|
+
rubygems_version: 2.6.12
|
76
91
|
signing_key:
|
77
92
|
specification_version: 4
|
78
93
|
summary: AWS plugin for Patriot Workflow Scheduler
|
79
94
|
test_files: []
|
80
|
-
has_rdoc:
|