logstash-input-sfdc_elf_schneider 1.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +11 -0
  3. data/.rubocop.yml +41 -0
  4. data/Gemfile +9 -0
  5. data/MIT-LICENSE +7 -0
  6. data/README.md +65 -0
  7. data/Rakefile +13 -0
  8. data/lib/logstash/inputs/sfdc_elf.rb +147 -0
  9. data/lib/logstash/inputs/sfdc_elf/client_with_streaming_support.rb +61 -0
  10. data/lib/logstash/inputs/sfdc_elf/queue_util.rb +176 -0
  11. data/lib/logstash/inputs/sfdc_elf/scheduler.rb +73 -0
  12. data/lib/logstash/inputs/sfdc_elf/state_persistor.rb +49 -0
  13. data/logstash-input-sfdc_elf_schneider.gemspec +29 -0
  14. data/spec/fixtures/auth_success_response.json +7 -0
  15. data/spec/fixtures/describe.json +3526 -0
  16. data/spec/fixtures/org_query_response.json +11 -0
  17. data/spec/fixtures/queue_util/create_event_ELF_list1.json +19 -0
  18. data/spec/fixtures/queue_util/create_event_ELF_list2.json +29 -0
  19. data/spec/fixtures/queue_util/create_event_ELF_list3.json +29 -0
  20. data/spec/fixtures/queue_util/create_event_sampledata1.csv +31 -0
  21. data/spec/fixtures/queue_util/create_event_sampledata2.csv +2 -0
  22. data/spec/fixtures/queue_util/create_event_sampledata3.csv +2 -0
  23. data/spec/fixtures/queue_util/eventlogfile_describe.json +990 -0
  24. data/spec/fixtures/queue_util/eventlogfile_list.json +62 -0
  25. data/spec/fixtures/queue_util/sample_data1.csv +5 -0
  26. data/spec/fixtures/queue_util/sample_data2.csv +5 -0
  27. data/spec/fixtures/queue_util/sample_data3.csv +1467 -0
  28. data/spec/fixtures/queue_util/sample_data4.csv +2 -0
  29. data/spec/fixtures/queue_util/sample_data5.csv +309 -0
  30. data/spec/inputs/sfdc_elf/queue_util_spec.rb +176 -0
  31. data/spec/inputs/sfdc_elf/scheduler_spec.rb +56 -0
  32. data/spec/inputs/sfdc_elf/state_persistor_spec.rb +58 -0
  33. data/spec/inputs/sfdc_elf_spec.rb +101 -0
  34. data/spec/spec_helper.rb +38 -0
  35. metadata +182 -0
@@ -0,0 +1,56 @@
1
+ require_relative '../../spec_helper'
2
+
3
+ describe Scheduler do
4
+ let(:hour_interval) { hour_interval = 3600 }
5
+
6
+ after do
7
+ Timecop.return
8
+ end
9
+
10
+ describe '#stall_schedule' do
11
+ # Precondition:
12
+ # - none
13
+ it 'Missed the next scheduled time' do
14
+ # Start time is 1:00pm.
15
+ start_time = Time.local(2015, 9, 1, 13, 00, 0)
16
+
17
+ # Freeze the time at the current time, which is 2:30pm.
18
+ current_time = Time.local(2015, 9, 1, 14, 30, 0)
19
+ Timecop.freeze(current_time)
20
+
21
+ # Set the interval in to every hour.
22
+ scheduler = Scheduler.new(hour_interval)
23
+
24
+ # The next schedule time should be based on the start time + the interval, which would be 2:00pm.
25
+ next_schedule_time = start_time + hour_interval
26
+
27
+ # Since the start time is 1:00pm and next schedule time is 2:00pm, but the current time is 2:30pm we missed the
28
+ # schedule time. So then we expect the there is no sleep and the next schedule is 3:00pm.
29
+ next_schedule_time = scheduler.stall_schedule(next_schedule_time)
30
+ expect(next_schedule_time).to eq Time.local(2015, 9, 1, 15, 00, 0)
31
+ end
32
+
33
+ # Precondition:
34
+ # - none
35
+ it 'Within the next scheduled time' do
36
+ # Start time is 1:00pm.
37
+ start_time = Time.local(2015, 9, 1, 13, 00, 0)
38
+
39
+ # Freeze the time at the current time, which is 1:30pm.
40
+ current_time = Time.local(2015, 9, 1, 13, 59, 55)
41
+ Timecop.freeze(current_time)
42
+
43
+ # Set the interval in to every hour.
44
+ scheduler = Scheduler.new(hour_interval)
45
+
46
+ # The next schedule time should be based on the start time + the interval, which would be 2:00pm.
47
+ next_schedule_time = start_time + hour_interval
48
+
49
+ # Since the start time is 1:00pm and next schedule time is 2:00pm, but the current time is 1:59:55pm, so we are
50
+ # within the scheduled time. So then we expect to sleep for 5 seconds and the next schedule is 3:00pm.
51
+ next_schedule_time = scheduler.stall_schedule(next_schedule_time)
52
+
53
+ expect(next_schedule_time).to eq Time.local(2015, 9, 1, 15, 00, 0)
54
+ end
55
+ end # stall_schedule
56
+ end # describe Scheduler
@@ -0,0 +1,58 @@
1
+ require_relative '../../spec_helper'
2
+
3
+ describe StatePersistor do
4
+ let(:provided_path) { provided_path = Dir.tmpdir }
5
+ let(:provided_org_id) { provided_org_id = 'some_org_id' }
6
+ let(:provided_path_with_file) { provided_path_with_file = "#{provided_path}/.sfdc_info_logstash_#{provided_org_id}" }
7
+
8
+ after do
9
+ # Delete the .sfdc_info_logstash file.
10
+ File.delete(provided_path_with_file) if File.exist?(provided_path_with_file)
11
+ end
12
+
13
+
14
+
15
+ describe '#get_last_indexed_log_date' do
16
+ # Precondition:
17
+ # - .sfdc_info_logstash file does not exist in the system temp directory.
18
+ it 'creates .sdfc_info_logstash file because it does not exist' do
19
+ # I expect the sfdc_info_logstash file to not exist.
20
+ expect(File.exist?(provided_path_with_file)).to eq false
21
+
22
+ # I expect the file to exist now.
23
+ state_persistor = StatePersistor.new(provided_path, provided_org_id)
24
+ state_persistor.get_last_indexed_log_date
25
+ expect(File.exist?(provided_path_with_file)).to eq true
26
+ end
27
+
28
+
29
+ # Precondition:
30
+ # - .sfdc_info_logstash file exist in the provided directory and with the default date in it which is
31
+ # created in this IT block.
32
+ it 'read from .sdfc_info_logstash file which as the default date in it' do
33
+ state_persistor = StatePersistor.new(provided_path, provided_org_id)
34
+
35
+ # I expect the .sfdc_info_logstash file to not exist with default date in it.
36
+ expect(state_persistor.get_last_indexed_log_date).to eq '0001-01-01T00:00:00Z'
37
+ end
38
+ end # get_last_indexed_log_date
39
+
40
+
41
+
42
+
43
+ describe '#update_last_indexed_log_date' do
44
+ # Precondition:
45
+ # - .sfdc_info_logstash file exist in the provided directory and with the default date in it which is
46
+ # created in this IT block.
47
+ it 'updates the last indexed LogDate on the .sdfc_info_logstash file' do
48
+ state_persistor = StatePersistor.new(provided_path, provided_org_id)
49
+
50
+ # I expect the .sfdc_info_logstash file having default date in it.
51
+ expect(state_persistor.get_last_indexed_log_date).to eq '0001-01-01T00:00:00Z'
52
+
53
+ # I expect the .sfdc_info_logstash file having the new date.
54
+ state_persistor.update_last_indexed_log_date('3672-21-11T23:59:342Z')
55
+ expect(state_persistor.get_last_indexed_log_date).to eq '3672-21-11T23:59:342Z'
56
+ end
57
+ end
58
+ end # describe StatePersistor
@@ -0,0 +1,101 @@
1
+ require_relative '../spec_helper'
2
+
3
+ describe LogStash::Inputs::SfdcElf do
4
+ describe 'Path config' do
5
+ let(:provided_path_with_file) { provided_path_with_file = "#{Dir.home}/.sfdc_info_logstash_ThisIsATestID00000" }
6
+
7
+ # Apply this stub_request to all test cases in this describe block because the suffix for the .sfdc_info_logstash
8
+ # is based on the the client org id, so a successful login is needed.
9
+ before do
10
+ # Stub authentication
11
+ stub_request(:post, /login.salesforce.com/).
12
+ with(headers: { Accept: '*/*', User_Agent: 'Ruby' }).
13
+ to_return(status: 200, body: fixture('auth_success_response.json'), headers: {})
14
+
15
+ # Stub organization query
16
+ stub_request(:get, 'https://na1.salesforce.com/services/data/v33.0/query?q=select%20id%20from%20Organization').
17
+ with(headers: { Accept: '*/*', Authorization: 'OAuth access_token', User_Agent: 'Ruby' }).
18
+ to_return(status: 200, body: fixture('org_query_response.json'), headers: {})
19
+
20
+ # Stub describe query
21
+ stub_request(:get, 'https://na1.salesforce.com/services/data/v33.0/sobjects/Organization/describe').
22
+ with(headers: { Accept: '*/*', Authorization: 'OAuth access_token', User_Agent: 'Ruby' }).
23
+ to_return(status: 200, body: fixture('describe.json'), headers: {})
24
+ end
25
+
26
+
27
+
28
+
29
+ # Precondition:
30
+ # - .sfdc_info_logstash file does not exist in the home directory.
31
+ # - Successful client login.
32
+ it 'sets .sfdc_info_logstash file in the home directory because no path was specified' do
33
+ config =
34
+ {
35
+ 'username' => 'me@example.com',
36
+ 'password' => 'password',
37
+ 'security_token' => 'security_token'
38
+ }
39
+
40
+ # Push config though the plugin life cycle of register and teardown only.
41
+ plugin = LogStash::Inputs::SfdcElf.new(config)
42
+ plugin.register
43
+ expect(plugin.path).to eq Dir.home
44
+ plugin.teardown
45
+
46
+ # Delete the .sfdc_info_logstash file.
47
+ File.delete(provided_path_with_file)
48
+ end
49
+
50
+
51
+
52
+
53
+ # Precondition:
54
+ # - .sfdc_info_logstash file does not exist in the home directory.
55
+ # - Successful client login.
56
+ it 'sets .sfdc_info_logstash file in the home directory because the provided path is does not exist' do
57
+ config =
58
+ {
59
+ 'username' => 'me@example.com',
60
+ 'password' => 'password',
61
+ 'security_token' => 'security_token',
62
+ 'path' => 'This/is/an/incorrect/path'
63
+ }
64
+
65
+ # Push config though the plugin life cycle of register and teardown only.
66
+ plugin = LogStash::Inputs::SfdcElf.new(config)
67
+ plugin.register
68
+ expect(plugin.path).to eq Dir.home
69
+ plugin.teardown
70
+
71
+ # Delete the .sfdc_info_logstash file.
72
+ File.delete(provided_path_with_file)
73
+ end
74
+
75
+
76
+
77
+
78
+ # Precondition:
79
+ # - .sfdc_info_logstash file does not exist in the provided directory.
80
+ # - Successful client login
81
+ it 'sets sfdc_info_path to the provided path' do
82
+ provided_path = Dir.tmpdir
83
+
84
+ config =
85
+ {
86
+ 'username' => 'me@example.com',
87
+ 'password' => 'password',
88
+ 'security_token' => 'security_token',
89
+ 'path' => provided_path
90
+ }
91
+
92
+ plugin = LogStash::Inputs::SfdcElf.new(config)
93
+ plugin.register
94
+ expect(plugin.path).to eq provided_path
95
+ plugin.teardown
96
+
97
+ # Delete the .sfdc_info_logstash file.
98
+ File.delete("#{provided_path}/.sfdc_info_logstash_ThisIsATestID00000")
99
+ end
100
+ end # Path for .sfdc_info_logstash
101
+ end # describe SfdcElf
@@ -0,0 +1,38 @@
1
+ # SimpleCov must be at the top of test class.
2
+ require 'simplecov'
3
+ SimpleCov.start
4
+
5
+ require 'logstash/devutils/rspec/spec_helper'
6
+ require 'lib/logstash/inputs/sfdc_elf'
7
+ require 'webmock/rspec'
8
+ require 'timecop'
9
+
10
+
11
+
12
+ # Set up color and formatting for Rspec tests.
13
+ RSpec.configure do |config|
14
+ # Use color in STDOUT
15
+ config.color = true
16
+
17
+ # Use color not only in STDOUT but also in pagers and files
18
+ config.tty = true
19
+
20
+ # Use the specified formatter
21
+ config.formatter = :documentation # :progress, :html, :textmate
22
+ end
23
+
24
+
25
+
26
+ # Turn off all connection to the internet.
27
+ WebMock.disable_net_connect!(allow_localhost: true)
28
+
29
+
30
+
31
+ # Helper methods to make it simple getting fixture data.
32
+ def fixture_path
33
+ File.expand_path('../fixtures', __FILE__)
34
+ end
35
+
36
+ def fixture(file)
37
+ File.new(fixture_path + '/' + file)
38
+ end
metadata ADDED
@@ -0,0 +1,182 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-input-sfdc_elf_schneider
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.5
5
+ platform: ruby
6
+ authors:
7
+ - Abhishek Sreenivasa
8
+ - Mohammed Islam
9
+ - Siddharatha Nagavarapu
10
+ autorequire:
11
+ bindir: bin
12
+ cert_chain: []
13
+ date: 2019-03-11 00:00:00.000000000 Z
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ requirement: !ruby/object:Gem::Requirement
17
+ requirements:
18
+ - - ">="
19
+ - !ruby/object:Gem::Version
20
+ version: 1.4.0
21
+ - - "<"
22
+ - !ruby/object:Gem::Version
23
+ version: 2.0.0
24
+ name: logstash-core
25
+ prerelease: false
26
+ type: :runtime
27
+ version_requirements: !ruby/object:Gem::Requirement
28
+ requirements:
29
+ - - ">="
30
+ - !ruby/object:Gem::Version
31
+ version: 1.4.0
32
+ - - "<"
33
+ - !ruby/object:Gem::Version
34
+ version: 2.0.0
35
+ - !ruby/object:Gem::Dependency
36
+ requirement: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ name: logstash-codec-plain
42
+ prerelease: false
43
+ type: :runtime
44
+ version_requirements: !ruby/object:Gem::Requirement
45
+ requirements:
46
+ - - ">="
47
+ - !ruby/object:Gem::Version
48
+ version: '0'
49
+ - !ruby/object:Gem::Dependency
50
+ requirement: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ name: activesupport
56
+ prerelease: false
57
+ type: :runtime
58
+ version_requirements: !ruby/object:Gem::Requirement
59
+ requirements:
60
+ - - ">="
61
+ - !ruby/object:Gem::Version
62
+ version: '0'
63
+ - !ruby/object:Gem::Dependency
64
+ requirement: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ name: databasedotcom
70
+ prerelease: false
71
+ type: :runtime
72
+ version_requirements: !ruby/object:Gem::Requirement
73
+ requirements:
74
+ - - ">="
75
+ - !ruby/object:Gem::Version
76
+ version: '0'
77
+ - !ruby/object:Gem::Dependency
78
+ requirement: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ name: logstash-devutils
84
+ prerelease: false
85
+ type: :development
86
+ version_requirements: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - ">="
89
+ - !ruby/object:Gem::Version
90
+ version: '0'
91
+ description: |-
92
+ This gem is a logstash plugin required to be installed on top of the Logstash core pipeline
93
+ using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program , changes made are quite specific to schneider electric, we removed some logtypes to support our cause
94
+ email:
95
+ - asreenivasa@salesforce.com
96
+ - mislam@salesforce.com
97
+ - siddharatha.nagavarapu@se.com
98
+ executables: []
99
+ extensions: []
100
+ extra_rdoc_files: []
101
+ files:
102
+ - ".gitignore"
103
+ - ".rubocop.yml"
104
+ - Gemfile
105
+ - MIT-LICENSE
106
+ - README.md
107
+ - Rakefile
108
+ - lib/logstash/inputs/sfdc_elf.rb
109
+ - lib/logstash/inputs/sfdc_elf/client_with_streaming_support.rb
110
+ - lib/logstash/inputs/sfdc_elf/queue_util.rb
111
+ - lib/logstash/inputs/sfdc_elf/scheduler.rb
112
+ - lib/logstash/inputs/sfdc_elf/state_persistor.rb
113
+ - logstash-input-sfdc_elf_schneider.gemspec
114
+ - spec/fixtures/auth_success_response.json
115
+ - spec/fixtures/describe.json
116
+ - spec/fixtures/org_query_response.json
117
+ - spec/fixtures/queue_util/create_event_ELF_list1.json
118
+ - spec/fixtures/queue_util/create_event_ELF_list2.json
119
+ - spec/fixtures/queue_util/create_event_ELF_list3.json
120
+ - spec/fixtures/queue_util/create_event_sampledata1.csv
121
+ - spec/fixtures/queue_util/create_event_sampledata2.csv
122
+ - spec/fixtures/queue_util/create_event_sampledata3.csv
123
+ - spec/fixtures/queue_util/eventlogfile_describe.json
124
+ - spec/fixtures/queue_util/eventlogfile_list.json
125
+ - spec/fixtures/queue_util/sample_data1.csv
126
+ - spec/fixtures/queue_util/sample_data2.csv
127
+ - spec/fixtures/queue_util/sample_data3.csv
128
+ - spec/fixtures/queue_util/sample_data4.csv
129
+ - spec/fixtures/queue_util/sample_data5.csv
130
+ - spec/inputs/sfdc_elf/queue_util_spec.rb
131
+ - spec/inputs/sfdc_elf/scheduler_spec.rb
132
+ - spec/inputs/sfdc_elf/state_persistor_spec.rb
133
+ - spec/inputs/sfdc_elf_spec.rb
134
+ - spec/spec_helper.rb
135
+ homepage: http://www.salesforce.com
136
+ licenses:
137
+ - MIT
138
+ metadata:
139
+ logstash_plugin: 'true'
140
+ logstash_group: input
141
+ post_install_message:
142
+ rdoc_options: []
143
+ require_paths:
144
+ - lib
145
+ required_ruby_version: !ruby/object:Gem::Requirement
146
+ requirements:
147
+ - - ">="
148
+ - !ruby/object:Gem::Version
149
+ version: '0'
150
+ required_rubygems_version: !ruby/object:Gem::Requirement
151
+ requirements:
152
+ - - ">="
153
+ - !ruby/object:Gem::Version
154
+ version: '0'
155
+ requirements: []
156
+ rubyforge_project:
157
+ rubygems_version: 2.7.6
158
+ signing_key:
159
+ specification_version: 4
160
+ summary: A Logstash plugin the receives events from Salesforce EventLogFile
161
+ test_files:
162
+ - spec/fixtures/auth_success_response.json
163
+ - spec/fixtures/describe.json
164
+ - spec/fixtures/org_query_response.json
165
+ - spec/fixtures/queue_util/create_event_ELF_list1.json
166
+ - spec/fixtures/queue_util/create_event_ELF_list2.json
167
+ - spec/fixtures/queue_util/create_event_ELF_list3.json
168
+ - spec/fixtures/queue_util/create_event_sampledata1.csv
169
+ - spec/fixtures/queue_util/create_event_sampledata2.csv
170
+ - spec/fixtures/queue_util/create_event_sampledata3.csv
171
+ - spec/fixtures/queue_util/eventlogfile_describe.json
172
+ - spec/fixtures/queue_util/eventlogfile_list.json
173
+ - spec/fixtures/queue_util/sample_data1.csv
174
+ - spec/fixtures/queue_util/sample_data2.csv
175
+ - spec/fixtures/queue_util/sample_data3.csv
176
+ - spec/fixtures/queue_util/sample_data4.csv
177
+ - spec/fixtures/queue_util/sample_data5.csv
178
+ - spec/inputs/sfdc_elf/queue_util_spec.rb
179
+ - spec/inputs/sfdc_elf/scheduler_spec.rb
180
+ - spec/inputs/sfdc_elf/state_persistor_spec.rb
181
+ - spec/inputs/sfdc_elf_spec.rb
182
+ - spec/spec_helper.rb