fluent-plugin-redshift 0.0.3 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -0
- data/VERSION +1 -1
- data/fluent-plugin-redshift.gemspec +1 -1
- data/lib/fluent/plugin/out_redshift.rb +9 -7
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1ebf84db40572de4663f8108e16d5c197dff045f
|
4
|
+
data.tar.gz: 5d07205c804fe6f677a0ca4d153f3ecbc91db248
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6a016ea6bb6870fed38611be2447eee1d131dcbb23f7027acb29c023c1cc0194a928666a9c852d2049b9128dfb1a64bda8d7b05ba28854141a82ae333c387fed
|
7
|
+
data.tar.gz: 5fdf7097033677ec8bfcbd0b05154555992ba7c6315fb5f35da455608e918ecff175fd4c8e3349e89cb531d994eacf2095cd47e8fb1e3347fbd6354a31082de9
|
data/README.md
CHANGED
@@ -113,6 +113,8 @@ Example (watch and upload json formatted apache log):
|
|
113
113
|
|
114
114
|
+ `redshift_schemaname` : schema name to store data. By default, this option is not set and find table without schema as your own search_path.
|
115
115
|
|
116
|
+
+ `redshift_connect_timeout` : maximum time to wait for connection to succeed.
|
117
|
+
|
116
118
|
+ `file_type` : file format of the source data. `csv`, `tsv`, `msgpack` or `json` are available.
|
117
119
|
|
118
120
|
+ `delimiter` : delimiter of the source data. This option will be ignored if `file_type` is specified.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.4
|
@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
|
|
18
18
|
|
19
19
|
gem.add_dependency "fluentd", "~> 0.10.0"
|
20
20
|
gem.add_dependency "aws-sdk", ">= 1.6.3"
|
21
|
-
gem.add_dependency "pg", "~> 0.
|
21
|
+
gem.add_dependency "pg", "~> 0.17.0"
|
22
22
|
gem.add_development_dependency "rake"
|
23
23
|
gem.add_development_dependency "simplecov", ">= 0.5.4"
|
24
24
|
gem.add_development_dependency "flexmock", ">= 1.3.1"
|
@@ -37,6 +37,7 @@ class RedshiftOutput < BufferedOutput
|
|
37
37
|
config_param :redshift_schemaname, :string, :default => nil
|
38
38
|
config_param :redshift_copy_base_options, :string , :default => "FILLRECORD ACCEPTANYDATE TRUNCATECOLUMNS"
|
39
39
|
config_param :redshift_copy_options, :string , :default => nil
|
40
|
+
config_param :redshift_connect_timeout, :integer, :default => 10
|
40
41
|
# file format
|
41
42
|
config_param :file_type, :string, :default => nil # json, tsv, csv, msgpack
|
42
43
|
config_param :delimiter, :string, :default => nil
|
@@ -53,7 +54,8 @@ class RedshiftOutput < BufferedOutput
|
|
53
54
|
port:@redshift_port,
|
54
55
|
dbname:@redshift_dbname,
|
55
56
|
user:@redshift_user,
|
56
|
-
password:@redshift_password
|
57
|
+
password:@redshift_password,
|
58
|
+
connect_timeout: @redshift_connect_timeout
|
57
59
|
}
|
58
60
|
@delimiter = determine_delimiter(@file_type) if @delimiter.nil? or @delimiter.empty?
|
59
61
|
$log.debug format_log("redshift file_type:#{@file_type} delimiter:'#{@delimiter}'")
|
@@ -113,8 +115,8 @@ class RedshiftOutput < BufferedOutput
|
|
113
115
|
# copy gz on s3 to redshift
|
114
116
|
s3_uri = "s3://#{@s3_bucket}/#{s3path}"
|
115
117
|
sql = @copy_sql_template % [s3_uri, @aws_sec_key]
|
116
|
-
$log.debug
|
117
|
-
|
118
|
+
$log.debug format_log("start copying. s3_uri=#{s3_uri}")
|
119
|
+
|
118
120
|
begin
|
119
121
|
conn = PG.connect(@db_conf)
|
120
122
|
conn.exec(sql)
|
@@ -175,9 +177,9 @@ class RedshiftOutput < BufferedOutput
|
|
175
177
|
gzw.write(tsv_text) if tsv_text and not tsv_text.empty?
|
176
178
|
rescue => e
|
177
179
|
if json?
|
178
|
-
$log.error format_log("failed to create table text from json. text=(#{record[@record_log_tag]})"), :error
|
180
|
+
$log.error format_log("failed to create table text from json. text=(#{record[@record_log_tag]})"), :error=>e.to_s
|
179
181
|
else
|
180
|
-
$log.error format_log("failed to create table text from msgpack. text=(#{record[@record_log_tag]})"), :error
|
182
|
+
$log.error format_log("failed to create table text from msgpack. text=(#{record[@record_log_tag]})"), :error=>e.to_s
|
181
183
|
end
|
182
184
|
|
183
185
|
$log.error_backtrace
|
@@ -202,15 +204,15 @@ class RedshiftOutput < BufferedOutput
|
|
202
204
|
end
|
203
205
|
|
204
206
|
def fetch_table_columns
|
205
|
-
conn = PG.connect(@db_conf)
|
206
207
|
begin
|
207
208
|
columns = nil
|
209
|
+
conn = PG.connect(@db_conf)
|
208
210
|
conn.exec(fetch_columns_sql_with_schema) do |result|
|
209
211
|
columns = result.collect{|row| row['column_name']}
|
210
212
|
end
|
211
213
|
columns
|
212
214
|
ensure
|
213
|
-
conn.close rescue nil
|
215
|
+
conn.close rescue nil if conn
|
214
216
|
end
|
215
217
|
end
|
216
218
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-redshift
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Masashi Miyazaki
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-03-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -44,14 +44,14 @@ dependencies:
|
|
44
44
|
requirements:
|
45
45
|
- - ~>
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: 0.
|
47
|
+
version: 0.17.0
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - ~>
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: 0.
|
54
|
+
version: 0.17.0
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: rake
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|