logstash-input-jdbc 3.1.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 6212e80eff0e3db1f5f86d14b2f3222f59a35dd6
4
- data.tar.gz: f23623cebfda1902907cb4f9fb5a460a73e362bc
3
+ metadata.gz: c06e48de482487475b4325cf2967edca7b6a8dc4
4
+ data.tar.gz: 5da108421a7003c5c6c0a4ac9d1b5d3425d50c15
5
5
  SHA512:
6
- metadata.gz: 4f19afa4d410887fc49f5ec8c5ac30e7ad7f345c79c7b17c1f3aed69654814d259d1e00205b668baa93ab30a4c88cf595fd3b9b84f44f8bcb133ecbca234b201
7
- data.tar.gz: cbda68232c8f6a7f5eddbfee91bc94d5a4f86f1a317912d85413de16f69e63e486bfa5c35ca1a804623c3b45e7021162fcea9f7fb76fae232a6de4505161a476
6
+ metadata.gz: eecfc3a25c9de60e40a47153aeb45d9386a96568a839d4ed01d1290eaf1163b1af9e096f760ad01f5ade5e3184b9d2ee4135e021b270b3a431c5af6fc573892b
7
+ data.tar.gz: 8f23e33c6ad652733238125d8c5d7d485a91d308c5c60aa15a8bc0e66adfe4a1faba8cb00b1b6f25b2569b0fc12be02138695ed96c4264e1207e73aa7a21cb1c
@@ -1,7 +1,5 @@
1
- # 3.1.0
2
- - Add an option to select the encoding data should be transform from, this will make sure all strings read from the jdbc connector are noremalized to be UTF-8 so no causing issues with later filters in LS.
3
- # 3.0.3
4
- - Added feature to read password from external file (#120).
1
+ ## 4.0.0
2
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
5
3
  # 3.0.2
6
4
  - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
7
5
  # 3.0.1
data/Gemfile CHANGED
@@ -1,2 +1,4 @@
1
1
  source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in logstash-mass_effect.gemspec
2
4
  gemspec
data/README.md CHANGED
@@ -95,7 +95,6 @@ Reading data from MySQL:
95
95
  jdbc_connection_string => "jdbc:mysql://host:port/database"
96
96
  jdbc_user => "user"
97
97
  jdbc_password => "password"
98
- # or jdbc_password_filepath => "/path/to/my/password_file"
99
98
  statement => "SELECT ..."
100
99
  jdbc_paging_enabled => "true"
101
100
  jdbc_page_size => "50000"
@@ -160,26 +160,6 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
160
160
  # Whether to force the lowercasing of identifier fields
161
161
  config :lowercase_column_names, :validate => :boolean, :default => true
162
162
 
163
- # The character encoding of all columns, leave empty if the columns are already properly UTF-8
164
- # encoded. Specific columns charsets using :columns_charset can override this setting.
165
- config :charset, :validate => :string
166
-
167
- # The character encoding for specific columns. This option will override the `:charset` option
168
- # for the specified columns.
169
- #
170
- # Example:
171
- # [source,ruby]
172
- # ----------------------------------
173
- # input {
174
- # jdbc {
175
- # ...
176
- # columns_charset => { "column0" => "ISO-8859-1" }
177
- # ...
178
- # }
179
- # }
180
- # this will only convert column0 that has ISO-8859-1 as an original encoding.
181
- config :columns_charset, :validate => :hash, :default => {}
182
-
183
163
  public
184
164
 
185
165
  def register
@@ -193,8 +173,6 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
193
173
  end
194
174
  end
195
175
 
196
- @enable_encoding = !@charset.nil? || !@columns_charset.empty?
197
-
198
176
  # load sql_last_value from file if exists
199
177
  if @clean_run && File.exist?(@last_run_metadata_path)
200
178
  File.delete(@last_run_metadata_path)
@@ -207,20 +185,6 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
207
185
  end
208
186
 
209
187
  @statement = File.read(@statement_filepath) if @statement_filepath
210
-
211
- if (@jdbc_password_filepath and @jdbc_password)
212
- raise(LogStash::ConfigurationError, "Only one of :jdbc_password, :jdbc_password_filepath may be set at a time.")
213
- end
214
-
215
- @jdbc_password = File.read(@jdbc_password_filepath).strip if @jdbc_password_filepath
216
-
217
- if enable_encoding?
218
- @converters = {}
219
- @columns_charset.each do |column_name, encoding|
220
- @converters[encoding] = LogStash::Util::Charset.new(encoding)
221
- end
222
- @converters[@charset] = LogStash::Util::Charset.new(@charset) if @charset
223
- end
224
188
  end # def register
225
189
 
226
190
  def run(queue)
@@ -250,10 +214,6 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
250
214
  # update default parameters
251
215
  @parameters['sql_last_value'] = @sql_last_value
252
216
  execute_statement(@statement, @parameters) do |row|
253
- if enable_encoding?
254
- ## do the necessary conversions to string elements
255
- row = Hash[row.map { |k, v| [k.to_s, convert(k, v)] }]
256
- end
257
217
  event = LogStash::Event.new(row)
258
218
  decorate(event)
259
219
  queue << event
@@ -266,24 +226,4 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
266
226
  end
267
227
  end
268
228
 
269
- private
270
-
271
- def enable_encoding?
272
- @enable_encoding
273
- end
274
-
275
- # make sure the encoding is uniform over fields
276
- def convert(column_name, value)
277
- return value unless value.is_a?(String)
278
- column_charset = @columns_charset[column_name]
279
- if column_charset
280
- converter = @converters[column_charset]
281
- converter.convert(value)
282
- elsif @charset
283
- converter = @converters[@charset]
284
- converter.convert(value)
285
- else
286
- value
287
- end
288
- end
289
229
  end # class LogStash::Inputs::Jdbc
@@ -40,9 +40,6 @@ module LogStash::PluginMixins::Jdbc
40
40
  # JDBC password
41
41
  config :jdbc_password, :validate => :password
42
42
 
43
- # JDBC password filename
44
- config :jdbc_password_filepath, :validate => :path
45
-
46
43
  # JDBC enable paging
47
44
  #
48
45
  # This will cause a sql statement to be broken up into multiple queries.
@@ -263,6 +260,7 @@ module LogStash::PluginMixins::Jdbc
263
260
 
264
261
  private
265
262
  def decorate_value(value)
263
+
266
264
  if value.is_a?(Time)
267
265
  # transform it to LogStash::Timestamp as required by LS
268
266
  LogStash::Timestamp.new(value)
@@ -271,7 +269,7 @@ module LogStash::PluginMixins::Jdbc
271
269
  # This is slower, so we put it in as a conditional case.
272
270
  LogStash::Timestamp.new(Time.parse(value.to_s))
273
271
  else
274
- value
272
+ value # no-op
275
273
  end
276
274
  end
277
275
  end
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-input-jdbc'
3
- s.version = '3.1.0'
3
+ s.version = '4.0.0'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "This example input streams a string at a definable interval."
6
6
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
18
18
  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
19
19
 
20
20
  # Gem dependencies
21
- s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
21
+ s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
22
22
  s.add_runtime_dependency 'logstash-codec-plain'
23
23
  s.add_runtime_dependency 'sequel'
24
24
  s.add_runtime_dependency 'tzinfo'
@@ -1,4 +1,3 @@
1
- # encoding: utf-8
2
1
  require "logstash/devutils/rspec/spec_helper"
3
2
  require "logstash/inputs/jdbc"
4
3
  require "jdbc/derby"
@@ -30,7 +29,6 @@ describe LogStash::Inputs::Jdbc do
30
29
  db.create_table :test_table do
31
30
  DateTime :created_at
32
31
  Integer :num
33
- String :string
34
32
  DateTime :custom_time
35
33
  end
36
34
  end
@@ -83,41 +81,6 @@ describe LogStash::Inputs::Jdbc do
83
81
  end
84
82
  end
85
83
 
86
- context "when both jdbc_password and jdbc_password_filepath arguments are passed" do
87
- let(:statement) { "SELECT * from test_table" }
88
- let(:jdbc_password) { "secret" }
89
- let(:jdbc_password_file_path) { Stud::Temporary.pathname }
90
- let(:settings) { { "jdbc_password_filepath" => jdbc_password_file_path,
91
- "jdbc_password" => jdbc_password,
92
- "statement" => statement } }
93
-
94
- it "should fail to register" do
95
- expect{ plugin.register }.to raise_error(LogStash::ConfigurationError)
96
- end
97
- end
98
-
99
- context "when jdbc_password is passed in from a file" do
100
- let(:statement) { "SELECT * from test_table" }
101
- let(:jdbc_password) { "secret" }
102
- let(:jdbc_password_file_path) { Stud::Temporary.pathname }
103
- let(:settings) { { "jdbc_password_filepath" => jdbc_password_file_path,
104
- "statement" => statement } }
105
-
106
- before do
107
- File.write(jdbc_password_file_path, jdbc_password)
108
- plugin.register
109
- end
110
-
111
- after do
112
- plugin.stop
113
- end
114
-
115
- it "should read in jdbc_password from file" do
116
- expect(plugin.jdbc_password).to eq(jdbc_password)
117
- end
118
- end
119
-
120
-
121
84
  context "when neither statement and statement_filepath arguments are passed" do
122
85
  it "should fail to register" do
123
86
  expect{ plugin.register }.to raise_error(LogStash::ConfigurationError)
@@ -171,7 +134,7 @@ describe LogStash::Inputs::Jdbc do
171
134
 
172
135
  it "should retrieve params correctly from Event" do
173
136
  plugin.run(queue)
174
- expect(queue.pop['num_param']).to eq(settings['parameters']['num_param'])
137
+ expect(queue.pop.get('num_param')).to eq(settings['parameters']['num_param'])
175
138
  end
176
139
  end
177
140
 
@@ -289,7 +252,7 @@ describe LogStash::Inputs::Jdbc do
289
252
  it "should convert it to LogStash::Timestamp " do
290
253
  plugin.run(queue)
291
254
  event = queue.pop
292
- expect(event["custom_time"]).to be_a(LogStash::Timestamp)
255
+ expect(event.get("custom_time")).to be_a(LogStash::Timestamp)
293
256
  end
294
257
  end
295
258
 
@@ -325,7 +288,7 @@ describe LogStash::Inputs::Jdbc do
325
288
  plugin.run(queue)
326
289
  event = queue.pop
327
290
  # This reflects a 6 hour time difference between UTC and America/Chicago
328
- expect(event["custom_time"].time).to eq(Time.iso8601("2015-01-01T18:00:00Z"))
291
+ expect(event.get("custom_time").time).to eq(Time.iso8601("2015-01-01T18:00:00Z"))
329
292
  end
330
293
  end
331
294
 
@@ -360,7 +323,7 @@ describe LogStash::Inputs::Jdbc do
360
323
  plugin.run(queue)
361
324
  event = queue.pop
362
325
  # With no timezone set, no change should occur
363
- expect(event["custom_time"].time).to eq(Time.iso8601("2015-01-01T12:00:00Z"))
326
+ expect(event.get("custom_time").time).to eq(Time.iso8601("2015-01-01T12:00:00Z"))
364
327
  end
365
328
  end
366
329
 
@@ -393,7 +356,7 @@ describe LogStash::Inputs::Jdbc do
393
356
 
394
357
  actual_sum = 0
395
358
  until queue.empty? do
396
- actual_sum += queue.pop['num']
359
+ actual_sum += queue.pop.get('num')
397
360
  end
398
361
 
399
362
  expect(actual_sum).to eq(nums.inject{|sum,x| sum + x })
@@ -874,109 +837,4 @@ describe LogStash::Inputs::Jdbc do
874
837
  plugin.stop
875
838
  end
876
839
  end
877
-
878
- context "when encoding of some columns need to be changed" do
879
-
880
- let(:settings) {{ "statement" => "SELECT * from test_table" }}
881
- let(:events) { [] }
882
- let(:row) do
883
- {
884
- "column0" => "foo",
885
- "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
886
- "column2" => 3
887
- }
888
- end
889
-
890
- before(:each) do
891
- allow_any_instance_of(Sequel::JDBC::Derby::Dataset).to receive(:each).and_yield(row)
892
- plugin.register
893
- end
894
-
895
- after(:each) do
896
- plugin.stop
897
- end
898
-
899
- it "should not convert any column by default" do
900
- encoded_row = {
901
- "column0" => "foo",
902
- "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
903
- "column2" => 3
904
- }
905
- expect(LogStash::Event).to receive(:new) do |row|
906
- row.each do |k, v|
907
- next unless v.is_a?(String)
908
- expect(row[k].encoding).to eq(encoded_row[k].encoding)
909
- end
910
- end
911
- plugin.run(events)
912
- end
913
-
914
- context "when all string columns should be encoded" do
915
-
916
- let(:settings) do
917
- {
918
- "statement" => "SELECT * from test_table",
919
- "charset" => "ISO-8859-1"
920
- }
921
- end
922
-
923
- let(:row) do
924
- {
925
- "column0" => "foo".force_encoding(Encoding::ISO_8859_1),
926
- "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
927
- "column2" => 3
928
- }
929
- end
930
-
931
- it "should transform all column string to UTF-8, default encoding" do
932
- encoded_row = {
933
- "column0" => "foo",
934
- "column1" => "bar",
935
- "column2" => 3
936
- }
937
- expect(LogStash::Event).to receive(:new) do |row|
938
- row.each do |k, v|
939
- next unless v.is_a?(String)
940
- expect(row[k].encoding).to eq(encoded_row[k].encoding)
941
- end
942
- end
943
- plugin.run(events)
944
- end
945
- end
946
-
947
- context "when only an specific column should be converted" do
948
-
949
- let(:settings) do
950
- {
951
- "statement" => "SELECT * from test_table",
952
- "columns_charset" => { "column1" => "ISO-8859-1" }
953
- }
954
- end
955
-
956
- let(:row) do
957
- {
958
- "column0" => "foo",
959
- "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
960
- "column2" => 3,
961
- "column3" => "berlin".force_encoding(Encoding::ASCII_8BIT)
962
- }
963
- end
964
-
965
- it "should only convert the selected column" do
966
- encoded_row = {
967
- "column0" => "foo",
968
- "column1" => "bar",
969
- "column2" => 3,
970
- "column3" => "berlin".force_encoding(Encoding::ASCII_8BIT)
971
- }
972
- expect(LogStash::Event).to receive(:new) do |row|
973
- row.each do |k, v|
974
- next unless v.is_a?(String)
975
- expect(row[k].encoding).to eq(encoded_row[k].encoding)
976
- end
977
- end
978
- plugin.run(events)
979
- end
980
- end
981
- end
982
840
  end
metadata CHANGED
@@ -1,156 +1,158 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-jdbc
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.1.0
4
+ version: 4.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-07-07 00:00:00.000000000 Z
11
+ date: 2016-05-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
+ name: logstash-core-plugin-api
14
15
  requirement: !ruby/object:Gem::Requirement
15
16
  requirements:
16
17
  - - "~>"
17
18
  - !ruby/object:Gem::Version
18
- version: '1.0'
19
- name: logstash-core-plugin-api
20
- prerelease: false
19
+ version: '2.0'
21
20
  type: :runtime
21
+ prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '1.0'
26
+ version: '2.0'
27
27
  - !ruby/object:Gem::Dependency
28
+ name: logstash-codec-plain
28
29
  requirement: !ruby/object:Gem::Requirement
29
30
  requirements:
30
31
  - - ">="
31
32
  - !ruby/object:Gem::Version
32
33
  version: '0'
33
- name: logstash-codec-plain
34
- prerelease: false
35
34
  type: :runtime
35
+ prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
41
  - !ruby/object:Gem::Dependency
42
+ name: sequel
42
43
  requirement: !ruby/object:Gem::Requirement
43
44
  requirements:
44
45
  - - ">="
45
46
  - !ruby/object:Gem::Version
46
47
  version: '0'
47
- name: sequel
48
- prerelease: false
49
48
  type: :runtime
49
+ prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
55
  - !ruby/object:Gem::Dependency
56
+ name: tzinfo
56
57
  requirement: !ruby/object:Gem::Requirement
57
58
  requirements:
58
59
  - - ">="
59
60
  - !ruby/object:Gem::Version
60
61
  version: '0'
61
- name: tzinfo
62
- prerelease: false
63
62
  type: :runtime
63
+ prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
66
  - - ">="
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0'
69
69
  - !ruby/object:Gem::Dependency
70
+ name: tzinfo-data
70
71
  requirement: !ruby/object:Gem::Requirement
71
72
  requirements:
72
73
  - - ">="
73
74
  - !ruby/object:Gem::Version
74
75
  version: '0'
75
- name: tzinfo-data
76
- prerelease: false
77
76
  type: :runtime
77
+ prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
80
  - - ">="
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
83
  - !ruby/object:Gem::Dependency
84
+ name: rufus-scheduler
84
85
  requirement: !ruby/object:Gem::Requirement
85
86
  requirements:
86
87
  - - ">="
87
88
  - !ruby/object:Gem::Version
88
89
  version: '0'
89
- name: rufus-scheduler
90
- prerelease: false
91
90
  type: :runtime
91
+ prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
94
  - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
97
  - !ruby/object:Gem::Dependency
98
+ name: logstash-devutils
98
99
  requirement: !ruby/object:Gem::Requirement
99
100
  requirements:
100
101
  - - ">="
101
102
  - !ruby/object:Gem::Version
102
103
  version: '0'
103
- name: logstash-devutils
104
- prerelease: false
105
104
  type: :development
105
+ prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
108
  - - ">="
109
109
  - !ruby/object:Gem::Version
110
110
  version: '0'
111
111
  - !ruby/object:Gem::Dependency
112
+ name: timecop
112
113
  requirement: !ruby/object:Gem::Requirement
113
114
  requirements:
114
115
  - - ">="
115
116
  - !ruby/object:Gem::Version
116
117
  version: '0'
117
- name: timecop
118
- prerelease: false
119
118
  type: :development
119
+ prerelease: false
120
120
  version_requirements: !ruby/object:Gem::Requirement
121
121
  requirements:
122
122
  - - ">="
123
123
  - !ruby/object:Gem::Version
124
124
  version: '0'
125
125
  - !ruby/object:Gem::Dependency
126
+ name: jdbc-derby
126
127
  requirement: !ruby/object:Gem::Requirement
127
128
  requirements:
128
129
  - - ">="
129
130
  - !ruby/object:Gem::Version
130
131
  version: '0'
131
- name: jdbc-derby
132
- prerelease: false
133
132
  type: :development
133
+ prerelease: false
134
134
  version_requirements: !ruby/object:Gem::Requirement
135
135
  requirements:
136
136
  - - ">="
137
137
  - !ruby/object:Gem::Version
138
138
  version: '0'
139
139
  - !ruby/object:Gem::Dependency
140
+ name: docker-api
140
141
  requirement: !ruby/object:Gem::Requirement
141
142
  requirements:
142
143
  - - ">="
143
144
  - !ruby/object:Gem::Version
144
145
  version: '0'
145
- name: docker-api
146
- prerelease: false
147
146
  type: :development
147
+ prerelease: false
148
148
  version_requirements: !ruby/object:Gem::Requirement
149
149
  requirements:
150
150
  - - ">="
151
151
  - !ruby/object:Gem::Version
152
152
  version: '0'
153
- description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
153
+ description: This gem is a Logstash plugin required to be installed on top of the
154
+ Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
155
+ gem is not a stand-alone program
154
156
  email: info@elastic.co
155
157
  executables: []
156
158
  extensions: []
@@ -171,7 +173,7 @@ licenses:
171
173
  metadata:
172
174
  logstash_plugin: 'true'
173
175
  logstash_group: input
174
- post_install_message:
176
+ post_install_message:
175
177
  rdoc_options: []
176
178
  require_paths:
177
179
  - lib
@@ -186,9 +188,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
186
188
  - !ruby/object:Gem::Version
187
189
  version: '0'
188
190
  requirements: []
189
- rubyforge_project:
190
- rubygems_version: 2.4.8
191
- signing_key:
191
+ rubyforge_project:
192
+ rubygems_version: 2.5.1
193
+ signing_key:
192
194
  specification_version: 4
193
195
  summary: This example input streams a string at a definable interval.
194
196
  test_files: