fluent-plugin-redshift 0.0.3 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0c2ea0e593b59bd6c6f6dae6f6b95fe496ec059e
4
- data.tar.gz: 3f7e925db5e64d32eaa446f0e969f32ce7137715
3
+ metadata.gz: 1ebf84db40572de4663f8108e16d5c197dff045f
4
+ data.tar.gz: 5d07205c804fe6f677a0ca4d153f3ecbc91db248
5
5
  SHA512:
6
- metadata.gz: 20247b545d6a63ecb4ba6b773338ae77416c01c4e567bcdedeb69b0668da7560b24ab91a844c702aa37c2aa74477038d4c00818dd17cba6804ce91155f642ef8
7
- data.tar.gz: 1a556e377f72c38430289154f98bf6f81da5737e7a09821153a9846fb5c67ad0878a5cce510513874d90b81dbfdee9370d2e3076be3ddc1f4dc53faf57633be1
6
+ metadata.gz: 6a016ea6bb6870fed38611be2447eee1d131dcbb23f7027acb29c023c1cc0194a928666a9c852d2049b9128dfb1a64bda8d7b05ba28854141a82ae333c387fed
7
+ data.tar.gz: 5fdf7097033677ec8bfcbd0b05154555992ba7c6315fb5f35da455608e918ecff175fd4c8e3349e89cb531d994eacf2095cd47e8fb1e3347fbd6354a31082de9
data/README.md CHANGED
@@ -113,6 +113,8 @@ Example (watch and upload json formatted apache log):
113
113
 
114
114
  + `redshift_schemaname` : schema name to store data. By default, this option is not set and find table without schema as your own search_path.
115
115
 
116
+ + `redshift_connect_timeout` : maximum time to wait for connection to succeed.
117
+
116
118
  + `file_type` : file format of the source data. `csv`, `tsv`, `msgpack` or `json` are available.
117
119
 
118
120
  + `delimiter` : delimiter of the source data. This option will be ignored if `file_type` is specified.
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.0.3
1
+ 0.0.4
@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
18
18
 
19
19
  gem.add_dependency "fluentd", "~> 0.10.0"
20
20
  gem.add_dependency "aws-sdk", ">= 1.6.3"
21
- gem.add_dependency "pg", "~> 0.14.0"
21
+ gem.add_dependency "pg", "~> 0.17.0"
22
22
  gem.add_development_dependency "rake"
23
23
  gem.add_development_dependency "simplecov", ">= 0.5.4"
24
24
  gem.add_development_dependency "flexmock", ">= 1.3.1"
@@ -37,6 +37,7 @@ class RedshiftOutput < BufferedOutput
37
37
  config_param :redshift_schemaname, :string, :default => nil
38
38
  config_param :redshift_copy_base_options, :string , :default => "FILLRECORD ACCEPTANYDATE TRUNCATECOLUMNS"
39
39
  config_param :redshift_copy_options, :string , :default => nil
40
+ config_param :redshift_connect_timeout, :integer, :default => 10
40
41
  # file format
41
42
  config_param :file_type, :string, :default => nil # json, tsv, csv, msgpack
42
43
  config_param :delimiter, :string, :default => nil
@@ -53,7 +54,8 @@ class RedshiftOutput < BufferedOutput
53
54
  port:@redshift_port,
54
55
  dbname:@redshift_dbname,
55
56
  user:@redshift_user,
56
- password:@redshift_password
57
+ password:@redshift_password,
58
+ connect_timeout: @redshift_connect_timeout
57
59
  }
58
60
  @delimiter = determine_delimiter(@file_type) if @delimiter.nil? or @delimiter.empty?
59
61
  $log.debug format_log("redshift file_type:#{@file_type} delimiter:'#{@delimiter}'")
@@ -113,8 +115,8 @@ class RedshiftOutput < BufferedOutput
113
115
  # copy gz on s3 to redshift
114
116
  s3_uri = "s3://#{@s3_bucket}/#{s3path}"
115
117
  sql = @copy_sql_template % [s3_uri, @aws_sec_key]
116
- $log.debug format_log("start copying. s3_uri=#{s3_uri}")
117
- conn = nil
118
+ $log.debug format_log("start copying. s3_uri=#{s3_uri}")
119
+
118
120
  begin
119
121
  conn = PG.connect(@db_conf)
120
122
  conn.exec(sql)
@@ -175,9 +177,9 @@ class RedshiftOutput < BufferedOutput
175
177
  gzw.write(tsv_text) if tsv_text and not tsv_text.empty?
176
178
  rescue => e
177
179
  if json?
178
- $log.error format_log("failed to create table text from json. text=(#{record[@record_log_tag]})"), :error=>$!.to_s
180
+ $log.error format_log("failed to create table text from json. text=(#{record[@record_log_tag]})"), :error=>e.to_s
179
181
  else
180
- $log.error format_log("failed to create table text from msgpack. text=(#{record[@record_log_tag]})"), :error=>$!.to_s
182
+ $log.error format_log("failed to create table text from msgpack. text=(#{record[@record_log_tag]})"), :error=>e.to_s
181
183
  end
182
184
 
183
185
  $log.error_backtrace
@@ -202,15 +204,15 @@ class RedshiftOutput < BufferedOutput
202
204
  end
203
205
 
204
206
  def fetch_table_columns
205
- conn = PG.connect(@db_conf)
206
207
  begin
207
208
  columns = nil
209
+ conn = PG.connect(@db_conf)
208
210
  conn.exec(fetch_columns_sql_with_schema) do |result|
209
211
  columns = result.collect{|row| row['column_name']}
210
212
  end
211
213
  columns
212
214
  ensure
213
- conn.close rescue nil
215
+ conn.close rescue nil if conn
214
216
  end
215
217
  end
216
218
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-redshift
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.3
4
+ version: 0.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Masashi Miyazaki
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-01-22 00:00:00.000000000 Z
11
+ date: 2014-03-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -44,14 +44,14 @@ dependencies:
44
44
  requirements:
45
45
  - - ~>
46
46
  - !ruby/object:Gem::Version
47
- version: 0.14.0
47
+ version: 0.17.0
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - ~>
53
53
  - !ruby/object:Gem::Version
54
- version: 0.14.0
54
+ version: 0.17.0
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: rake
57
57
  requirement: !ruby/object:Gem::Requirement