fluent-plugin-redshift-auto 0.0.2 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/lib/fluent/plugin/out_redshift_auto.rb +26 -5
- metadata +3 -3
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.4
|
@@ -34,6 +34,8 @@ class RedshiftOutput < BufferedOutput
|
|
34
34
|
config_param :redshift_user, :string
|
35
35
|
config_param :redshift_password, :string
|
36
36
|
config_param :redshift_tablename, :string
|
37
|
+
config_param :redshift_schemaname, :string, :default => "public"
|
38
|
+
config_param :make_auto_table, :integer, :default => 1 #1 => make_auto 0=> no
|
37
39
|
# file format
|
38
40
|
config_param :file_type, :string, :default => nil # json, tsv, csv
|
39
41
|
config_param :delimiter, :string, :default => nil
|
@@ -56,7 +58,7 @@ class RedshiftOutput < BufferedOutput
|
|
56
58
|
}
|
57
59
|
@delimiter = determine_delimiter(@file_type) if @delimiter.nil? or @delimiter.empty?
|
58
60
|
$log.debug format_log("redshift file_type:#{@file_type} delimiter:'#{@delimiter}'")
|
59
|
-
@copy_sql_template = "copy #{@redshift_tablename} from '%s' CREDENTIALS 'aws_access_key_id=#{@aws_key_id};aws_secret_access_key=%s' delimiter '#{@delimiter}' GZIP TRUNCATECOLUMNS ESCAPE FILLRECORD ACCEPTANYDATE;"
|
61
|
+
@copy_sql_template = "copy #{@redshift_schemaname}.#{@redshift_tablename} from '%s' CREDENTIALS 'aws_access_key_id=#{@aws_key_id};aws_secret_access_key=%s' delimiter '#{@delimiter}' GZIP TRUNCATECOLUMNS ESCAPE FILLRECORD ACCEPTANYDATE;"
|
60
62
|
end
|
61
63
|
|
62
64
|
def start
|
@@ -74,7 +76,7 @@ class RedshiftOutput < BufferedOutput
|
|
74
76
|
|
75
77
|
def format(tag, time, record)
|
76
78
|
record = JSON.generate(record)
|
77
|
-
if json?
|
79
|
+
if @make_auto_table == 1 && json?
|
78
80
|
json = JSON.parse(record)
|
79
81
|
cols = []
|
80
82
|
json.each do |key,val|
|
@@ -189,7 +191,7 @@ class RedshiftOutput < BufferedOutput
|
|
189
191
|
end
|
190
192
|
|
191
193
|
def fetch_table_columns
|
192
|
-
fetch_columns_sql = "select column_name from INFORMATION_SCHEMA.COLUMNS where table_name = '#{@redshift_tablename}' order by ordinal_position;"
|
194
|
+
fetch_columns_sql = "select column_name from INFORMATION_SCHEMA.COLUMNS where table_name = '#{@redshift_tablename}' and table_schema = '#{@redshift_schemaname}' order by ordinal_position;"
|
193
195
|
conn = PG.connect(@db_conf)
|
194
196
|
begin
|
195
197
|
columns = nil
|
@@ -274,10 +276,29 @@ class RedshiftOutput < BufferedOutput
|
|
274
276
|
len = cols.length
|
275
277
|
cols.slice!(len - 1)
|
276
278
|
|
277
|
-
if @redshift_schemaname
|
279
|
+
if @redshift_schemaname && @redshift_schemaname != "public"
|
280
|
+
sql = "SELECT nspname FROM pg_namespace WHERE nspname LIKE '#{@redshift_schemaname}';"
|
281
|
+
cnt = 0
|
282
|
+
conn.exec(sql).each do |r|
|
283
|
+
cnt = cnt + 1
|
284
|
+
end
|
285
|
+
|
286
|
+
if cnt == 0
|
287
|
+
sql = "CREATE SCHEMA #{@redshift_schemaname}"
|
288
|
+
begin
|
289
|
+
conn.exec(sql)
|
290
|
+
rescue PGError => e
|
291
|
+
$log.error format_log("failed CREATE SCHEMA schema_name: #{@redshift_schemaname}")
|
292
|
+
$log.error format_log("class: " + e.class + " msg: " + e.message)
|
293
|
+
rescue => e
|
294
|
+
$log.error format_log("failed CREATE SCHEMA schema_name: #{@redshift_schemaname}")
|
295
|
+
$log.error format_log("class: " + e.class + " msg: " + e.message)
|
296
|
+
end
|
297
|
+
$log.info format_log("SCHEMA CREATED: => #{sql}")
|
298
|
+
end
|
278
299
|
table_name = "#{@redshift_schemaname}.#{tag}"
|
279
300
|
else
|
280
|
-
table_name = "#{tag}"
|
301
|
+
table_name = "#{tag}"
|
281
302
|
end
|
282
303
|
|
283
304
|
sql = "CREATE TABLE #{table_name} (#{cols});"
|
metadata
CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
5
5
|
segments:
|
6
6
|
- 0
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
version: 0.0.
|
8
|
+
- 4
|
9
|
+
version: 0.0.4
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Takashi Honda
|
@@ -14,7 +14,7 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date: 2013-10-
|
17
|
+
date: 2013-10-21 00:00:00 +09:00
|
18
18
|
default_executable:
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|