logstash-input-jdbc 4.0.2 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: afede6bb954e753352fe3138057cb32dbffc814d
4
- data.tar.gz: 624feecfdfdea557f91b480acfccb7d0948a4660
3
+ metadata.gz: e7ea26c9ab4cd30fa89736ea255dfa9794dd5ab3
4
+ data.tar.gz: deaad338731a1da19e117942c27516477c1e3307
5
5
  SHA512:
6
- metadata.gz: da164fbd6abf2627372925ff198ba99fd822588392365498f3543a928e1ffcdd297b3098ba30ac4d8426b06059f4c4fde8cbef4ef1f21652a9345298e27c641c
7
- data.tar.gz: 4b6a2ede22c3392d6553cb2b04a7b1c26c5333722ffc7dc16ebe0fa19eee49dce7bdee9a773a4ce35aff55692942d7db06950ea571d72d37f94e165e66ea846a
6
+ metadata.gz: c29cf8b2fbf3e2120f6cfd94b28665fb4282443fc468a6a148373a642de6a4ccac9fc141b8ebef84e3cd2d396b11b1013ff025ccc09982e33d1bf89963ac4f50
7
+ data.tar.gz: 28f23e618871d17d77125a918f9787ad92a3cc4f05094b57fcf93e0cf082ba34f443a641b99939fea61689206bfcf6775e3265ae2bc49335789b4474d53b3f99
@@ -1,9 +1,13 @@
1
- ## 4.0.2
2
- - Added feature to read password from external file (#120).
1
+ ## 4.1.0
2
+ - Add an option to select the encoding data should be transform from,
3
+ this will make sure all strings read from the jdbc connector are
4
+ noremalized to be UTF-8 so no causing issues with later filters in LS.
3
5
  ## 4.0.1
4
6
  - Republish all the gems under jruby.
5
7
  ## 4.0.0
6
8
  - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
9
+ # 3.0.3
10
+ - Added feature to read password from external file (#120)
7
11
  # 3.0.2
8
12
  - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
9
13
  # 3.0.1
@@ -160,6 +160,26 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
160
160
  # Whether to force the lowercasing of identifier fields
161
161
  config :lowercase_column_names, :validate => :boolean, :default => true
162
162
 
163
+ # The character encoding of all columns, leave empty if the columns are already properly UTF-8
164
+ # encoded. Specific columns charsets using :columns_charset can override this setting.
165
+ config :charset, :validate => :string
166
+
167
+ # The character encoding for specific columns. This option will override the `:charset` option
168
+ # for the specified columns.
169
+ #
170
+ # Example:
171
+ # [source,ruby]
172
+ # ----------------------------------
173
+ # input {
174
+ # jdbc {
175
+ # ...
176
+ # columns_charset => { "column0" => "ISO-8859-1" }
177
+ # ...
178
+ # }
179
+ # }
180
+ # this will only convert column0 that has ISO-8859-1 as an original encoding.
181
+ config :columns_charset, :validate => :hash, :default => {}
182
+
163
183
  public
164
184
 
165
185
  def register
@@ -173,6 +193,8 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
173
193
  end
174
194
  end
175
195
 
196
+ @enable_encoding = !@charset.nil? || !@columns_charset.empty?
197
+
176
198
  # load sql_last_value from file if exists
177
199
  if @clean_run && File.exist?(@last_run_metadata_path)
178
200
  File.delete(@last_run_metadata_path)
@@ -191,6 +213,14 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
191
213
  end
192
214
 
193
215
  @jdbc_password = File.read(@jdbc_password_filepath).strip if @jdbc_password_filepath
216
+
217
+ if enable_encoding?
218
+ @converters = {}
219
+ @columns_charset.each do |column_name, encoding|
220
+ @converters[encoding] = LogStash::Util::Charset.new(encoding)
221
+ end
222
+ @converters[@charset] = LogStash::Util::Charset.new(@charset) if @charset
223
+ end
194
224
  end # def register
195
225
 
196
226
  def run(queue)
@@ -220,6 +250,10 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
220
250
  # update default parameters
221
251
  @parameters['sql_last_value'] = @sql_last_value
222
252
  execute_statement(@statement, @parameters) do |row|
253
+ if enable_encoding?
254
+ ## do the necessary conversions to string elements
255
+ row = Hash[row.map { |k, v| [k.to_s, convert(k, v)] }]
256
+ end
223
257
  event = LogStash::Event.new(row)
224
258
  decorate(event)
225
259
  queue << event
@@ -232,4 +266,24 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
232
266
  end
233
267
  end
234
268
 
269
+ private
270
+
271
+ def enable_encoding?
272
+ @enable_encoding
273
+ end
274
+
275
+ # make sure the encoding is uniform over fields
276
+ def convert(column_name, value)
277
+ return value unless value.is_a?(String)
278
+ column_charset = @columns_charset[column_name]
279
+ if column_charset
280
+ converter = @converters[column_charset]
281
+ converter.convert(value)
282
+ elsif @charset
283
+ converter = @converters[@charset]
284
+ converter.convert(value)
285
+ else
286
+ value
287
+ end
288
+ end
235
289
  end # class LogStash::Inputs::Jdbc
@@ -263,7 +263,6 @@ module LogStash::PluginMixins::Jdbc
263
263
 
264
264
  private
265
265
  def decorate_value(value)
266
-
267
266
  if value.is_a?(Time)
268
267
  # transform it to LogStash::Timestamp as required by LS
269
268
  LogStash::Timestamp.new(value)
@@ -272,7 +271,7 @@ module LogStash::PluginMixins::Jdbc
272
271
  # This is slower, so we put it in as a conditional case.
273
272
  LogStash::Timestamp.new(Time.parse(value.to_s))
274
273
  else
275
- value # no-op
274
+ value
276
275
  end
277
276
  end
278
277
  end
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-input-jdbc'
3
- s.version = '4.0.2'
3
+ s.version = '4.1.0'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "This example input streams a string at a definable interval."
6
6
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -1,3 +1,4 @@
1
+ # encoding: utf-8
1
2
  require "logstash/devutils/rspec/spec_helper"
2
3
  require "logstash/inputs/jdbc"
3
4
  require "jdbc/derby"
@@ -29,6 +30,7 @@ describe LogStash::Inputs::Jdbc do
29
30
  db.create_table :test_table do
30
31
  DateTime :created_at
31
32
  Integer :num
33
+ String :string
32
34
  DateTime :custom_time
33
35
  end
34
36
  end
@@ -301,29 +303,51 @@ describe LogStash::Inputs::Jdbc do
301
303
 
302
304
  let(:settings) do
303
305
  {
304
- "statement" => "SELECT * from test_table",
306
+ "statement" => "SELECT * from test_table WHERE custom_time > :sql_last_value",
307
+ "use_column_value" => true,
308
+ "tracking_column" => "custom_time",
309
+ "last_run_metadata_path" => Stud::Temporary.pathname
305
310
  }
306
311
  end
307
312
 
308
- let(:num_rows) { 10 }
313
+ let(:hour_range) { 10..20 }
309
314
 
310
- before do
311
- num_rows.times do
312
- db[:test_table].insert(:num => 1, :custom_time => "2015-01-01 12:00:00", :created_at => Time.now.utc)
315
+ it "should convert the time to reflect the timezone " do
316
+ last_run_value = Time.iso8601("2000-01-01T00:00:00.000Z")
317
+ File.write(settings["last_run_metadata_path"], YAML.dump(last_run_value))
318
+
319
+ hour_range.each do |i|
320
+ db[:test_table].insert(:num => i, :custom_time => "2015-01-01 #{i}:00:00", :created_at => Time.now.utc)
313
321
  end
314
322
 
315
323
  plugin.register
316
- end
317
324
 
318
- after do
325
+ plugin.run(queue)
326
+ expected = ["2015-01-01T16:00:00.000Z",
327
+ "2015-01-01T17:00:00.000Z",
328
+ "2015-01-01T18:00:00.000Z",
329
+ "2015-01-01T19:00:00.000Z",
330
+ "2015-01-01T20:00:00.000Z",
331
+ "2015-01-01T21:00:00.000Z",
332
+ "2015-01-01T22:00:00.000Z",
333
+ "2015-01-01T23:00:00.000Z",
334
+ "2015-01-02T00:00:00.000Z",
335
+ "2015-01-02T01:00:00.000Z",
336
+ "2015-01-02T02:00:00.000Z"].map { |i| Time.iso8601(i) }
337
+ actual = queue.size.times.map { queue.pop.get("custom_time").time }
338
+ expect(actual).to eq(expected)
319
339
  plugin.stop
320
- end
321
340
 
322
- it "should convert the time to reflect the timezone " do
323
341
  plugin.run(queue)
342
+ expect(queue.size).to eq(0)
343
+ db[:test_table].insert(:num => 11, :custom_time => "2015-01-01 11:00:00", :created_at => Time.now.utc)
344
+ db[:test_table].insert(:num => 12, :custom_time => "2015-01-01 21:00:00", :created_at => Time.now.utc)
345
+ plugin.run(queue)
346
+ expect(queue.size).to eq(1)
324
347
  event = queue.pop
325
- # This reflects a 6 hour time difference between UTC and America/Chicago
326
- expect(event.get("custom_time").time).to eq(Time.iso8601("2015-01-01T18:00:00Z"))
348
+ expect(event.get("num")).to eq(12)
349
+ expect(event.get("custom_time").time).to eq(Time.iso8601("2015-01-02T03:00:00.000Z"))
350
+ p settings
327
351
  end
328
352
  end
329
353
 
@@ -872,4 +896,109 @@ describe LogStash::Inputs::Jdbc do
872
896
  plugin.stop
873
897
  end
874
898
  end
899
+
900
+ context "when encoding of some columns need to be changed" do
901
+
902
+ let(:settings) {{ "statement" => "SELECT * from test_table" }}
903
+ let(:events) { [] }
904
+ let(:row) do
905
+ {
906
+ "column0" => "foo",
907
+ "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
908
+ "column2" => 3
909
+ }
910
+ end
911
+
912
+ before(:each) do
913
+ allow_any_instance_of(Sequel::JDBC::Derby::Dataset).to receive(:each).and_yield(row)
914
+ plugin.register
915
+ end
916
+
917
+ after(:each) do
918
+ plugin.stop
919
+ end
920
+
921
+ it "should not convert any column by default" do
922
+ encoded_row = {
923
+ "column0" => "foo",
924
+ "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
925
+ "column2" => 3
926
+ }
927
+ expect(LogStash::Event).to receive(:new) do |row|
928
+ row.each do |k, v|
929
+ next unless v.is_a?(String)
930
+ expect(row[k].encoding).to eq(encoded_row[k].encoding)
931
+ end
932
+ end
933
+ plugin.run(events)
934
+ end
935
+
936
+ context "when all string columns should be encoded" do
937
+
938
+ let(:settings) do
939
+ {
940
+ "statement" => "SELECT * from test_table",
941
+ "charset" => "ISO-8859-1"
942
+ }
943
+ end
944
+
945
+ let(:row) do
946
+ {
947
+ "column0" => "foo".force_encoding(Encoding::ISO_8859_1),
948
+ "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
949
+ "column2" => 3
950
+ }
951
+ end
952
+
953
+ it "should transform all column string to UTF-8, default encoding" do
954
+ encoded_row = {
955
+ "column0" => "foo",
956
+ "column1" => "bar",
957
+ "column2" => 3
958
+ }
959
+ expect(LogStash::Event).to receive(:new) do |row|
960
+ row.each do |k, v|
961
+ next unless v.is_a?(String)
962
+ expect(row[k].encoding).to eq(encoded_row[k].encoding)
963
+ end
964
+ end
965
+ plugin.run(events)
966
+ end
967
+ end
968
+
969
+ context "when only an specific column should be converted" do
970
+
971
+ let(:settings) do
972
+ {
973
+ "statement" => "SELECT * from test_table",
974
+ "columns_charset" => { "column1" => "ISO-8859-1" }
975
+ }
976
+ end
977
+
978
+ let(:row) do
979
+ {
980
+ "column0" => "foo",
981
+ "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
982
+ "column2" => 3,
983
+ "column3" => "berlin".force_encoding(Encoding::ASCII_8BIT)
984
+ }
985
+ end
986
+
987
+ it "should only convert the selected column" do
988
+ encoded_row = {
989
+ "column0" => "foo",
990
+ "column1" => "bar",
991
+ "column2" => 3,
992
+ "column3" => "berlin".force_encoding(Encoding::ASCII_8BIT)
993
+ }
994
+ expect(LogStash::Event).to receive(:new) do |row|
995
+ row.each do |k, v|
996
+ next unless v.is_a?(String)
997
+ expect(row[k].encoding).to eq(encoded_row[k].encoding)
998
+ end
999
+ end
1000
+ plugin.run(events)
1001
+ end
1002
+ end
1003
+ end
875
1004
  end
metadata CHANGED
@@ -1,155 +1,155 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-jdbc
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.0.2
4
+ version: 4.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-05-18 00:00:00.000000000 Z
11
+ date: 2016-07-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: logstash-core-plugin-api
15
- version_requirements: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - ~>
18
- - !ruby/object:Gem::Version
19
- version: '2.0'
20
14
  requirement: !ruby/object:Gem::Requirement
21
15
  requirements:
22
- - - ~>
16
+ - - "~>"
23
17
  - !ruby/object:Gem::Version
24
18
  version: '2.0'
19
+ name: logstash-core-plugin-api
25
20
  prerelease: false
26
21
  type: :runtime
27
- - !ruby/object:Gem::Dependency
28
- name: logstash-codec-plain
29
22
  version_requirements: !ruby/object:Gem::Requirement
30
23
  requirements:
31
- - - '>='
24
+ - - "~>"
32
25
  - !ruby/object:Gem::Version
33
- version: '0'
26
+ version: '2.0'
27
+ - !ruby/object:Gem::Dependency
34
28
  requirement: !ruby/object:Gem::Requirement
35
29
  requirements:
36
- - - '>='
30
+ - - ">="
37
31
  - !ruby/object:Gem::Version
38
32
  version: '0'
33
+ name: logstash-codec-plain
39
34
  prerelease: false
40
35
  type: :runtime
41
- - !ruby/object:Gem::Dependency
42
- name: sequel
43
36
  version_requirements: !ruby/object:Gem::Requirement
44
37
  requirements:
45
- - - '>='
38
+ - - ">="
46
39
  - !ruby/object:Gem::Version
47
40
  version: '0'
41
+ - !ruby/object:Gem::Dependency
48
42
  requirement: !ruby/object:Gem::Requirement
49
43
  requirements:
50
- - - '>='
44
+ - - ">="
51
45
  - !ruby/object:Gem::Version
52
46
  version: '0'
47
+ name: sequel
53
48
  prerelease: false
54
49
  type: :runtime
55
- - !ruby/object:Gem::Dependency
56
- name: tzinfo
57
50
  version_requirements: !ruby/object:Gem::Requirement
58
51
  requirements:
59
- - - '>='
52
+ - - ">="
60
53
  - !ruby/object:Gem::Version
61
54
  version: '0'
55
+ - !ruby/object:Gem::Dependency
62
56
  requirement: !ruby/object:Gem::Requirement
63
57
  requirements:
64
- - - '>='
58
+ - - ">="
65
59
  - !ruby/object:Gem::Version
66
60
  version: '0'
61
+ name: tzinfo
67
62
  prerelease: false
68
63
  type: :runtime
69
- - !ruby/object:Gem::Dependency
70
- name: tzinfo-data
71
64
  version_requirements: !ruby/object:Gem::Requirement
72
65
  requirements:
73
- - - '>='
66
+ - - ">="
74
67
  - !ruby/object:Gem::Version
75
68
  version: '0'
69
+ - !ruby/object:Gem::Dependency
76
70
  requirement: !ruby/object:Gem::Requirement
77
71
  requirements:
78
- - - '>='
72
+ - - ">="
79
73
  - !ruby/object:Gem::Version
80
74
  version: '0'
75
+ name: tzinfo-data
81
76
  prerelease: false
82
77
  type: :runtime
83
- - !ruby/object:Gem::Dependency
84
- name: rufus-scheduler
85
78
  version_requirements: !ruby/object:Gem::Requirement
86
79
  requirements:
87
- - - '>='
80
+ - - ">="
88
81
  - !ruby/object:Gem::Version
89
82
  version: '0'
83
+ - !ruby/object:Gem::Dependency
90
84
  requirement: !ruby/object:Gem::Requirement
91
85
  requirements:
92
- - - '>='
86
+ - - ">="
93
87
  - !ruby/object:Gem::Version
94
88
  version: '0'
89
+ name: rufus-scheduler
95
90
  prerelease: false
96
91
  type: :runtime
97
- - !ruby/object:Gem::Dependency
98
- name: logstash-devutils
99
92
  version_requirements: !ruby/object:Gem::Requirement
100
93
  requirements:
101
- - - '>='
94
+ - - ">="
102
95
  - !ruby/object:Gem::Version
103
96
  version: '0'
97
+ - !ruby/object:Gem::Dependency
104
98
  requirement: !ruby/object:Gem::Requirement
105
99
  requirements:
106
- - - '>='
100
+ - - ">="
107
101
  - !ruby/object:Gem::Version
108
102
  version: '0'
103
+ name: logstash-devutils
109
104
  prerelease: false
110
105
  type: :development
111
- - !ruby/object:Gem::Dependency
112
- name: timecop
113
106
  version_requirements: !ruby/object:Gem::Requirement
114
107
  requirements:
115
- - - '>='
108
+ - - ">="
116
109
  - !ruby/object:Gem::Version
117
110
  version: '0'
111
+ - !ruby/object:Gem::Dependency
118
112
  requirement: !ruby/object:Gem::Requirement
119
113
  requirements:
120
- - - '>='
114
+ - - ">="
121
115
  - !ruby/object:Gem::Version
122
116
  version: '0'
117
+ name: timecop
123
118
  prerelease: false
124
119
  type: :development
125
- - !ruby/object:Gem::Dependency
126
- name: jdbc-derby
127
120
  version_requirements: !ruby/object:Gem::Requirement
128
121
  requirements:
129
- - - '>='
122
+ - - ">="
130
123
  - !ruby/object:Gem::Version
131
124
  version: '0'
125
+ - !ruby/object:Gem::Dependency
132
126
  requirement: !ruby/object:Gem::Requirement
133
127
  requirements:
134
- - - '>='
128
+ - - ">="
135
129
  - !ruby/object:Gem::Version
136
130
  version: '0'
131
+ name: jdbc-derby
137
132
  prerelease: false
138
133
  type: :development
139
- - !ruby/object:Gem::Dependency
140
- name: docker-api
141
134
  version_requirements: !ruby/object:Gem::Requirement
142
135
  requirements:
143
- - - '>='
136
+ - - ">="
144
137
  - !ruby/object:Gem::Version
145
138
  version: '0'
139
+ - !ruby/object:Gem::Dependency
146
140
  requirement: !ruby/object:Gem::Requirement
147
141
  requirements:
148
- - - '>='
142
+ - - ">="
149
143
  - !ruby/object:Gem::Version
150
144
  version: '0'
145
+ name: docker-api
151
146
  prerelease: false
152
147
  type: :development
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - ">="
151
+ - !ruby/object:Gem::Version
152
+ version: '0'
153
153
  description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
154
154
  email: info@elastic.co
155
155
  executables: []
@@ -177,12 +177,12 @@ require_paths:
177
177
  - lib
178
178
  required_ruby_version: !ruby/object:Gem::Requirement
179
179
  requirements:
180
- - - '>='
180
+ - - ">="
181
181
  - !ruby/object:Gem::Version
182
182
  version: '0'
183
183
  required_rubygems_version: !ruby/object:Gem::Requirement
184
184
  requirements:
185
- - - '>='
185
+ - - ">="
186
186
  - !ruby/object:Gem::Version
187
187
  version: '0'
188
188
  requirements: []