fluent-plugin-redshift-auto 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/lib/fluent/plugin/out_redshift_auto.rb +5 -2
- metadata +3 -3
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
0.0.
|
|
1
|
+
0.0.5
|
|
@@ -58,7 +58,7 @@ class RedshiftOutput < BufferedOutput
|
|
|
58
58
|
}
|
|
59
59
|
@delimiter = determine_delimiter(@file_type) if @delimiter.nil? or @delimiter.empty?
|
|
60
60
|
$log.debug format_log("redshift file_type:#{@file_type} delimiter:'#{@delimiter}'")
|
|
61
|
-
@copy_sql_template = "copy #{@redshift_schemaname}
|
|
61
|
+
@copy_sql_template = "copy #{@redshift_schemaname}.%s from '%s' CREDENTIALS 'aws_access_key_id=#{@aws_key_id};aws_secret_access_key=%s' delimiter '#{@delimiter}' GZIP TRUNCATECOLUMNS ESCAPE FILLRECORD ACCEPTANYDATE;"
|
|
62
62
|
end
|
|
63
63
|
|
|
64
64
|
def start
|
|
@@ -90,6 +90,8 @@ class RedshiftOutput < BufferedOutput
|
|
|
90
90
|
|
|
91
91
|
def write(chunk)
|
|
92
92
|
$log.debug format_log("start creating gz.")
|
|
93
|
+
file_name = File::basename(chunk.path)
|
|
94
|
+
table_name = file_name.sub(/\..*/, "")
|
|
93
95
|
|
|
94
96
|
# create a gz file
|
|
95
97
|
tmp = Tempfile.new("s3-")
|
|
@@ -110,7 +112,8 @@ class RedshiftOutput < BufferedOutput
|
|
|
110
112
|
:acl => :bucket_owner_full_control)
|
|
111
113
|
# copy gz on s3 to redshift
|
|
112
114
|
s3_uri = "s3://#{@s3_bucket}/#{s3path}"
|
|
113
|
-
sql = @copy_sql_template % [s3_uri, @aws_sec_key]
|
|
115
|
+
sql = @copy_sql_template % [table_name, s3_uri, @aws_sec_key]
|
|
116
|
+
$log.error format_log(sql)
|
|
114
117
|
$log.debug format_log("start copying. s3_uri=#{s3_uri}")
|
|
115
118
|
conn = nil
|
|
116
119
|
begin
|
metadata
CHANGED
|
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
|
5
5
|
segments:
|
|
6
6
|
- 0
|
|
7
7
|
- 0
|
|
8
|
-
-
|
|
9
|
-
version: 0.0.
|
|
8
|
+
- 5
|
|
9
|
+
version: 0.0.5
|
|
10
10
|
platform: ruby
|
|
11
11
|
authors:
|
|
12
12
|
- Takashi Honda
|
|
@@ -14,7 +14,7 @@ autorequire:
|
|
|
14
14
|
bindir: bin
|
|
15
15
|
cert_chain: []
|
|
16
16
|
|
|
17
|
-
date: 2013-
|
|
17
|
+
date: 2013-12-06 00:00:00 +09:00
|
|
18
18
|
default_executable:
|
|
19
19
|
dependencies:
|
|
20
20
|
- !ruby/object:Gem::Dependency
|