fluent-plugin-redshift-out 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/lib/fluent/plugin/{out_redshift.rb → out_redshift-out.rb} +13 -8
- metadata +2 -2
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.2.0
|
@@ -25,7 +25,7 @@ class RedshiftOutput < BufferedOutput
|
|
25
25
|
config_param :aws_key_id, :string
|
26
26
|
config_param :aws_sec_key, :string
|
27
27
|
config_param :s3_bucket, :string
|
28
|
-
config_param :
|
28
|
+
config_param :s3_region, :string, :default => nil
|
29
29
|
config_param :path, :string, :default => ""
|
30
30
|
config_param :timestamp_key_format, :string, :default => 'year=%Y/month=%m/day=%d/hour=%H/%Y%m%d-%H%M'
|
31
31
|
config_param :utc, :bool, :default => false
|
@@ -61,17 +61,17 @@ class RedshiftOutput < BufferedOutput
|
|
61
61
|
}
|
62
62
|
@delimiter = determine_delimiter(@file_type) if @delimiter.nil? or @delimiter.empty?
|
63
63
|
$log.debug format_log("redshift file_type:#{@file_type} delimiter:'#{@delimiter}'")
|
64
|
-
@copy_sql_template = "copy #{table_name_with_schema} from '%s' CREDENTIALS 'aws_access_key_id=#{@aws_key_id};aws_secret_access_key=%s' delimiter '#{@delimiter}' GZIP ESCAPE #{@redshift_copy_base_options} #{@redshift_copy_options};"
|
65
64
|
end
|
66
65
|
|
67
66
|
def start
|
68
67
|
super
|
69
68
|
# init s3 conf
|
70
|
-
options = {
|
71
|
-
|
72
|
-
:
|
73
|
-
|
74
|
-
|
69
|
+
options = {}
|
70
|
+
if @aws_key_id && @aws_sec_key
|
71
|
+
options[:access_key_id] = @aws_key_id
|
72
|
+
options[:secret_access_key] = @aws_sec_key
|
73
|
+
end
|
74
|
+
options[:region] = @s3_region if @s3_region
|
75
75
|
@s3 = AWS::S3.new(options)
|
76
76
|
@bucket = @s3.buckets[@s3_bucket]
|
77
77
|
end
|
@@ -116,7 +116,12 @@ class RedshiftOutput < BufferedOutput
|
|
116
116
|
|
117
117
|
# copy gz on s3 to redshift
|
118
118
|
s3_uri = "s3://#{@s3_bucket}/#{s3path}"
|
119
|
-
|
119
|
+
credentials = @s3.client.credential_provider.credentials
|
120
|
+
sql = "copy #{table_name_with_schema} from '#{s3_uri}'"
|
121
|
+
sql += " CREDENTIALS 'aws_access_key_id=#{credentials[:access_key_id]}; aws_secret_access_key=#{credentials[:secret_access_key]};"
|
122
|
+
sql += " token=#{credentials[:session_token]};" if credentials[:session_token]
|
123
|
+
sql += " delimiter '#{@delimiter}' GZIP ESCAPE #{@redshift_copy_base_options} #{@redshift_copy_options};"
|
124
|
+
|
120
125
|
$log.debug format_log("start copying. s3_uri=#{s3_uri}")
|
121
126
|
|
122
127
|
begin
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-redshift-out
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -152,7 +152,7 @@ files:
|
|
152
152
|
- Rakefile
|
153
153
|
- VERSION
|
154
154
|
- fluent-plugin-redshift-out.gemspec
|
155
|
-
- lib/fluent/plugin/out_redshift.rb
|
155
|
+
- lib/fluent/plugin/out_redshift-out.rb
|
156
156
|
- test/plugin/test_out_redshift.rb
|
157
157
|
- test/test_helper.rb
|
158
158
|
homepage: https://github.com/sakuma/fluent-plugin-redshift-out
|