fluent-plugin-redshift-out 0.1.0 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.0
1
+ 0.2.0
@@ -25,7 +25,7 @@ class RedshiftOutput < BufferedOutput
25
25
  config_param :aws_key_id, :string
26
26
  config_param :aws_sec_key, :string
27
27
  config_param :s3_bucket, :string
28
- config_param :s3_endpoint, :string, :default => nil
28
+ config_param :s3_region, :string, :default => nil
29
29
  config_param :path, :string, :default => ""
30
30
  config_param :timestamp_key_format, :string, :default => 'year=%Y/month=%m/day=%d/hour=%H/%Y%m%d-%H%M'
31
31
  config_param :utc, :bool, :default => false
@@ -61,17 +61,17 @@ class RedshiftOutput < BufferedOutput
61
61
  }
62
62
  @delimiter = determine_delimiter(@file_type) if @delimiter.nil? or @delimiter.empty?
63
63
  $log.debug format_log("redshift file_type:#{@file_type} delimiter:'#{@delimiter}'")
64
- @copy_sql_template = "copy #{table_name_with_schema} from '%s' CREDENTIALS 'aws_access_key_id=#{@aws_key_id};aws_secret_access_key=%s' delimiter '#{@delimiter}' GZIP ESCAPE #{@redshift_copy_base_options} #{@redshift_copy_options};"
65
64
  end
66
65
 
67
66
  def start
68
67
  super
69
68
  # init s3 conf
70
- options = {
71
- :access_key_id => @aws_key_id,
72
- :secret_access_key => @aws_sec_key
73
- }
74
- options[:s3_endpoint] = @s3_endpoint if @s3_endpoint
69
+ options = {}
70
+ if @aws_key_id && @aws_sec_key
71
+ options[:access_key_id] = @aws_key_id
72
+ options[:secret_access_key] = @aws_sec_key
73
+ end
74
+ options[:region] = @s3_region if @s3_region
75
75
  @s3 = AWS::S3.new(options)
76
76
  @bucket = @s3.buckets[@s3_bucket]
77
77
  end
@@ -116,7 +116,12 @@ class RedshiftOutput < BufferedOutput
116
116
 
117
117
  # copy gz on s3 to redshift
118
118
  s3_uri = "s3://#{@s3_bucket}/#{s3path}"
119
- sql = @copy_sql_template % [s3_uri, @aws_sec_key]
119
+ credentials = @s3.client.credential_provider.credentials
120
+ sql = "copy #{table_name_with_schema} from '#{s3_uri}'"
121
+ sql += " CREDENTIALS 'aws_access_key_id=#{credentials[:access_key_id]}; aws_secret_access_key=#{credentials[:secret_access_key]};"
122
+ sql += " token=#{credentials[:session_token]};" if credentials[:session_token]
123
+ sql += " delimiter '#{@delimiter}' GZIP ESCAPE #{@redshift_copy_base_options} #{@redshift_copy_options};"
124
+
120
125
  $log.debug format_log("start copying. s3_uri=#{s3_uri}")
121
126
 
122
127
  begin
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-redshift-out
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -152,7 +152,7 @@ files:
152
152
  - Rakefile
153
153
  - VERSION
154
154
  - fluent-plugin-redshift-out.gemspec
155
- - lib/fluent/plugin/out_redshift.rb
155
+ - lib/fluent/plugin/out_redshift-out.rb
156
156
  - test/plugin/test_out_redshift.rb
157
157
  - test/test_helper.rb
158
158
  homepage: https://github.com/sakuma/fluent-plugin-redshift-out