patriot-aws 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -13
- data/lib/patriot_aws/command.rb +1 -0
- data/lib/patriot_aws/command/redshift.rb +121 -0
- data/lib/patriot_aws/command/s3.rb +0 -0
- data/lib/patriot_aws/version.rb +1 -1
- metadata +24 -10
checksums.yaml
CHANGED
@@ -1,15 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
|
5
|
-
data.tar.gz: !binary |-
|
6
|
-
ZGMwYmJhNDkwZGIwYzhlYjdjZjM2NTQwZjNlNDljNmUyMDBmMzVlYg==
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 26b16fcbab39dffb44c42a4badeb96236a388835
|
4
|
+
data.tar.gz: 7dcafac13fbbcfd9b701dc296697e99669058edd
|
7
5
|
SHA512:
|
8
|
-
metadata.gz:
|
9
|
-
|
10
|
-
ODBlZmZjMDM1OWIxNDQxZjhlOGM0YzEyM2IzZWJiOGE1YjRmNTdhMzA2OGEx
|
11
|
-
NjQzZTdiMWI1YWYzM2EyODcwODg1ZTdlNWZkNGM4ZGMyNzJhNWE=
|
12
|
-
data.tar.gz: !binary |-
|
13
|
-
ODZmNTI4OWY4OWM1MzdjMDI0YzM0MWRhYjhmYWEwMmM4MGFhMjc5NTllMWRm
|
14
|
-
ODRhOTE2Zjg4NDNiYjY1NDY1NTg0ZTMwMWU2OTMyYmU1N2QxMTMzNjBkOGIz
|
15
|
-
OTkyOTU4ODgxMjBhNWNkMGM5OWY1MDhjZDAzZTM2OTkyZjRkMjE=
|
6
|
+
metadata.gz: 780e3b20dfa8cb4aec12935a29bbf378df59639ab5c5f981e7e343adce9ddcb2421c0bdcca2facf051e8912244b20b749329e12a9e859d52a22208c6f2462376
|
7
|
+
data.tar.gz: 450bd34d82813b575a2b66acff47d20c3fe3e7bc3bbd3e1bb19f11ccb88acb1b135d562c234396ce7355da2236d6c8687780f8eeea34346f30b30dc0bac99184
|
data/lib/patriot_aws/command.rb
CHANGED
@@ -0,0 +1,121 @@
|
|
1
|
+
=begin
|
2
|
+
|
3
|
+
sample_select.pbc
|
4
|
+
--
|
5
|
+
redshift {
|
6
|
+
name "select_from_redshift"
|
7
|
+
name_suffix _date_
|
8
|
+
inifile '/path/to//redshift.ini'
|
9
|
+
query 'select * from test_table'
|
10
|
+
options :with_header => true, :delimiter => "\t"
|
11
|
+
}
|
12
|
+
|
13
|
+
sample_copy.pbc
|
14
|
+
--
|
15
|
+
redshift {
|
16
|
+
name "s3_to_redshift"
|
17
|
+
name_suffix _date_
|
18
|
+
inifile '/Users/b06959/tmp/redshift.ini'
|
19
|
+
query <<-EOS
|
20
|
+
COPY #{schema}.#{table}
|
21
|
+
FROM '#{s3_path}'
|
22
|
+
ACCESS_KEY_ID '%{access_key_id}'
|
23
|
+
SECRET_ACCESS_KEY '%{secret_access_key}'
|
24
|
+
delimiter '\t'
|
25
|
+
gzip
|
26
|
+
EOS
|
27
|
+
}
|
28
|
+
|
29
|
+
$ cat redshift.ini
|
30
|
+
[connection]
|
31
|
+
host = staging.xxxxxxxxxxxxx.ap-northeast-1.redshift.amazonaws.com
|
32
|
+
user = staging
|
33
|
+
password = staging
|
34
|
+
dbname = staging
|
35
|
+
port = 5439
|
36
|
+
|
37
|
+
[s3credentials]
|
38
|
+
access_key_id = #{access_key_id},
|
39
|
+
secret_access_key = #{secret_access_key}
|
40
|
+
=end
|
41
|
+
|
42
|
+
require 'pg'
|
43
|
+
|
44
|
+
module PatriotAWS
|
45
|
+
module Command
|
46
|
+
class RedshiftCommand < Patriot::Command::Base
|
47
|
+
declare_command_name :redshift
|
48
|
+
include PatriotAWS::Ext::AWS
|
49
|
+
|
50
|
+
command_attr :name, :name_suffix, :inifile, :options, :query
|
51
|
+
|
52
|
+
def job_id
|
53
|
+
job_id = "#{command_name}_#{@name}_#{@name_suffix}"
|
54
|
+
job_id
|
55
|
+
end
|
56
|
+
|
57
|
+
# @see Patriot::Command::Base#configure
|
58
|
+
def configure
|
59
|
+
@name_suffix ||= _date_
|
60
|
+
self
|
61
|
+
end
|
62
|
+
|
63
|
+
def execute
|
64
|
+
@logger.info 'start redshift query...'
|
65
|
+
@options ||= {}
|
66
|
+
|
67
|
+
ini = IniFile.load(@inifile)
|
68
|
+
raise Exception, 'inifile not found.' if ini.nil?
|
69
|
+
raise Exception, 'query is not set.' if @query.nil?
|
70
|
+
|
71
|
+
_set_options
|
72
|
+
|
73
|
+
begin
|
74
|
+
# replace variables
|
75
|
+
if ini['s3credentials']
|
76
|
+
@query = @query % ini['s3credentials'].symbolize_keys
|
77
|
+
end
|
78
|
+
|
79
|
+
conn = PG::Connection.new(
|
80
|
+
ini['connection'].symbolize_keys
|
81
|
+
)
|
82
|
+
res = conn.exec(@query)
|
83
|
+
|
84
|
+
output_arr = Array.new
|
85
|
+
if res then
|
86
|
+
res.each_with_index do |r, idx|
|
87
|
+
# print header
|
88
|
+
if @options[:with_header] && idx == 0
|
89
|
+
output_arr.push r.keys.join(@options[:delimiter])
|
90
|
+
end
|
91
|
+
|
92
|
+
output_arr.push r.values.join(@options[:delimiter])
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
puts output_arr.join("\n")
|
97
|
+
# rescue PGError => ex
|
98
|
+
# # may cause PGError when connection setting is invalid
|
99
|
+
# # e.g.
|
100
|
+
# # PG::ConnectionBad -> could not translate host name "test" to address: nodename nor servname provided, or not known
|
101
|
+
# print(ex.class," -> ",ex.message)
|
102
|
+
#
|
103
|
+
# raise PGError
|
104
|
+
# rescue => ex
|
105
|
+
# # Other Error process
|
106
|
+
# print(ex.class," -> ",ex.message)
|
107
|
+
ensure
|
108
|
+
conn.close if conn
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
# @private
|
113
|
+
# set default option parameters
|
114
|
+
def _set_options
|
115
|
+
@options[:with_header] = false if @options[:with_header].nil?
|
116
|
+
@options[:delimiter] = "\t" if @options[:delimiter].nil?
|
117
|
+
end
|
118
|
+
private :_set_options
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
File without changes
|
data/lib/patriot_aws/version.rb
CHANGED
metadata
CHANGED
@@ -1,41 +1,55 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: patriot-aws
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Takayuki Tanaka
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-06-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - ~>
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '2'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - ~>
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '2'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: pg
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.18.4
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 0.18.4
|
27
41
|
- !ruby/object:Gem::Dependency
|
28
42
|
name: patriot-workflow-scheduler
|
29
43
|
requirement: !ruby/object:Gem::Requirement
|
30
44
|
requirements:
|
31
|
-
- - ~>
|
45
|
+
- - "~>"
|
32
46
|
- !ruby/object:Gem::Version
|
33
47
|
version: '0.7'
|
34
48
|
type: :runtime
|
35
49
|
prerelease: false
|
36
50
|
version_requirements: !ruby/object:Gem::Requirement
|
37
51
|
requirements:
|
38
|
-
- - ~>
|
52
|
+
- - "~>"
|
39
53
|
- !ruby/object:Gem::Version
|
40
54
|
version: '0.7'
|
41
55
|
description: plugins for Patriot Workflow Scheduler, which deal with AWS such as S3.
|
@@ -48,6 +62,7 @@ files:
|
|
48
62
|
- init.rb
|
49
63
|
- lib/patriot_aws.rb
|
50
64
|
- lib/patriot_aws/command.rb
|
65
|
+
- lib/patriot_aws/command/redshift.rb
|
51
66
|
- lib/patriot_aws/command/s3.rb
|
52
67
|
- lib/patriot_aws/ext.rb
|
53
68
|
- lib/patriot_aws/ext/aws.rb
|
@@ -62,19 +77,18 @@ require_paths:
|
|
62
77
|
- lib
|
63
78
|
required_ruby_version: !ruby/object:Gem::Requirement
|
64
79
|
requirements:
|
65
|
-
- -
|
80
|
+
- - ">="
|
66
81
|
- !ruby/object:Gem::Version
|
67
82
|
version: '0'
|
68
83
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
69
84
|
requirements:
|
70
|
-
- -
|
85
|
+
- - ">="
|
71
86
|
- !ruby/object:Gem::Version
|
72
87
|
version: '0'
|
73
88
|
requirements: []
|
74
89
|
rubyforge_project: patriot-aws
|
75
|
-
rubygems_version: 2.
|
90
|
+
rubygems_version: 2.6.12
|
76
91
|
signing_key:
|
77
92
|
specification_version: 4
|
78
93
|
summary: AWS plugin for Patriot Workflow Scheduler
|
79
94
|
test_files: []
|
80
|
-
has_rdoc:
|