logstash-input-sfdc_elf 1.0.2

data/spec/inputs/sfdc_elf/scheduler_spec.rb

@@ -0,0 +1,56 @@
+require_relative '../../spec_helper'
+
+describe Scheduler do
+  let(:hour_interval) { hour_interval = 3600 }
+
+  after do
+    Timecop.return
+  end
+
+  describe '#stall_schedule' do
+    # Precondition:
+    #               - none
+    it 'Missed the next scheduled time' do
+      # Start time is 1:00pm.
+      start_time = Time.local(2015, 9, 1, 13, 00, 0)
+
+      # Freeze the time at the current time, which is 2:30pm.
+      current_time = Time.local(2015, 9, 1, 14, 30, 0)
+      Timecop.freeze(current_time)
+
+      # Set the interval in to every hour.
+      scheduler = Scheduler.new(hour_interval)
+
+      # The next schedule time should be based on the start time + the interval, which would be 2:00pm.
+      next_schedule_time = start_time + hour_interval
+
+      # Since the start time is 1:00pm and next schedule time is 2:00pm, but the current time is 2:30pm we missed the
+      # schedule time. So then we expect the there is no sleep and the next schedule is 3:00pm.
+      next_schedule_time = scheduler.stall_schedule(next_schedule_time)
+      expect(next_schedule_time).to eq Time.local(2015, 9, 1, 15, 00, 0)
+    end
+
+    # Precondition:
+    #               - none
+    it 'Within the next scheduled time' do
+      # Start time is 1:00pm.
+      start_time = Time.local(2015, 9, 1, 13, 00, 0)
+
+      # Freeze the time at the current time, which is 1:30pm.
+      current_time = Time.local(2015, 9, 1, 13, 59, 55)
+      Timecop.freeze(current_time)
+
+      # Set the interval in to every hour.
+      scheduler = Scheduler.new(hour_interval)
+
+      # The next schedule time should be based on the start time + the interval, which would be 2:00pm.
+      next_schedule_time = start_time + hour_interval
+
+      # Since the start time is 1:00pm and next schedule time is 2:00pm, but the current time is 1:59:55pm, so we are
+      # within the scheduled time. So then we expect to sleep for 5 seconds and the next schedule is 3:00pm.
+      next_schedule_time = scheduler.stall_schedule(next_schedule_time)
+
+      expect(next_schedule_time).to eq Time.local(2015, 9, 1, 15, 00, 0)
+    end
+  end # stall_schedule
+end # describe Scheduler

data/spec/inputs/sfdc_elf/state_persistor_spec.rb

@@ -0,0 +1,58 @@
+require_relative '../../spec_helper'
+
+describe StatePersistor do
+  let(:provided_path) { provided_path =  Dir.tmpdir }
+  let(:provided_org_id) { provided_org_id =  'some_org_id' }
+  let(:provided_path_with_file) { provided_path_with_file =  "#{provided_path}/.sfdc_info_logstash_#{provided_org_id}" }
+
+  after do
+    # Delete the .sfdc_info_logstash file.
+    File.delete(provided_path_with_file) if File.exist?(provided_path_with_file)
+  end
+
+
+
+  describe '#get_last_indexed_log_date' do
+    # Precondition:
+    #               - .sfdc_info_logstash file does not exist in the system temp directory.
+    it 'creates .sdfc_info_logstash file because it does not exist' do
+      # I expect the sfdc_info_logstash file to not exist.
+      expect(File.exist?(provided_path_with_file)).to eq false
+
+      # I expect the file to exist now.
+      state_persistor = StatePersistor.new(provided_path, provided_org_id)
+      state_persistor.get_last_indexed_log_date
+      expect(File.exist?(provided_path_with_file)).to eq true
+    end
+
+
+    # Precondition:
+    #               - .sfdc_info_logstash file exist in the provided directory and with the default date in it which is
+    #                 created in this IT block.
+    it 'read from .sdfc_info_logstash file which as the default date in it' do
+      state_persistor = StatePersistor.new(provided_path, provided_org_id)
+
+      # I expect the .sfdc_info_logstash file to not exist with default date in it.
+      expect(state_persistor.get_last_indexed_log_date).to eq '0001-01-01T00:00:00Z'
+    end
+  end # get_last_indexed_log_date
+
+
+
+
+  describe '#update_last_indexed_log_date' do
+    # Precondition:
+    #               - .sfdc_info_logstash file exist in the provided directory and with the default date in it which is
+    #                 created in this IT block.
+    it 'updates the last indexed LogDate on the .sdfc_info_logstash file' do
+      state_persistor = StatePersistor.new(provided_path, provided_org_id)
+
+      # I expect the .sfdc_info_logstash file having default date in it.
+      expect(state_persistor.get_last_indexed_log_date).to eq '0001-01-01T00:00:00Z'
+
+      # I expect the .sfdc_info_logstash file having the new date.
+      state_persistor.update_last_indexed_log_date('3672-21-11T23:59:342Z')
+      expect(state_persistor.get_last_indexed_log_date).to eq '3672-21-11T23:59:342Z'
+    end
+  end
+end # describe StatePersistor

data/spec/inputs/sfdc_elf_spec.rb

@@ -0,0 +1,101 @@
+require_relative '../spec_helper'
+
+describe LogStash::Inputs::SfdcElf do
+  describe 'Path config' do
+    let(:provided_path_with_file) { provided_path_with_file =  "#{Dir.home}/.sfdc_info_logstash_ThisIsATestID00000" }
+
+    # Apply this stub_request to all test cases in this describe block because the suffix for the .sfdc_info_logstash
+    # is based on the the client org id, so a successful login is needed.
+    before do
+      # Stub authentication
+      stub_request(:post, /login.salesforce.com/).
+        with(headers: { Accept: '*/*', User_Agent: 'Ruby' }).
+        to_return(status: 200, body: fixture('auth_success_response.json'), headers: {})
+
+      # Stub organization query
+      stub_request(:get, 'https://na1.salesforce.com/services/data/v33.0/query?q=select%20id%20from%20Organization').
+        with(headers: { Accept: '*/*', Authorization: 'OAuth access_token', User_Agent: 'Ruby' }).
+        to_return(status: 200, body: fixture('org_query_response.json'), headers: {})
+
+      # Stub describe query
+      stub_request(:get, 'https://na1.salesforce.com/services/data/v33.0/sobjects/Organization/describe').
+        with(headers: { Accept: '*/*', Authorization: 'OAuth access_token', User_Agent: 'Ruby' }).
+        to_return(status: 200, body: fixture('describe.json'), headers: {})
+    end
+
+
+
+
+    # Precondition:
+    #               - .sfdc_info_logstash file does not exist in the home directory.
+    #               - Successful client login.
+    it 'sets .sfdc_info_logstash file in the home directory because no path was specified' do
+      config =
+        {
+          'username' => 'me@example.com',
+          'password' => 'password',
+          'security_token' => 'security_token'
+        }
+
+      # Push config though the plugin life cycle of register and teardown only.
+      plugin = LogStash::Inputs::SfdcElf.new(config)
+      plugin.register
+      expect(plugin.path).to eq Dir.home
+      plugin.teardown
+
+      # Delete the .sfdc_info_logstash file.
+      File.delete(provided_path_with_file)
+    end
+
+
+
+
+    # Precondition:
+    #               - .sfdc_info_logstash file does not exist in the home directory.
+    #               - Successful client login.
+    it 'sets .sfdc_info_logstash file in the home directory because the provided path is does not exist' do
+      config =
+        {
+          'username' => 'me@example.com',
+          'password' => 'password',
+          'security_token' => 'security_token',
+          'path' => 'This/is/an/incorrect/path'
+        }
+
+      # Push config though the plugin life cycle of register and teardown only.
+      plugin = LogStash::Inputs::SfdcElf.new(config)
+      plugin.register
+      expect(plugin.path).to eq Dir.home
+      plugin.teardown
+
+      # Delete the .sfdc_info_logstash file.
+      File.delete(provided_path_with_file)
+    end
+
+
+
+
+    # Precondition:
+    #               - .sfdc_info_logstash file does not exist in the provided directory.
+    #               - Successful client login
+    it 'sets sfdc_info_path to the provided path' do
+      provided_path = Dir.tmpdir
+
+      config =
+        {
+          'username' => 'me@example.com',
+          'password' => 'password',
+          'security_token' => 'security_token',
+          'path' => provided_path
+        }
+
+      plugin = LogStash::Inputs::SfdcElf.new(config)
+      plugin.register
+      expect(plugin.path).to eq provided_path
+      plugin.teardown
+
+      # Delete the .sfdc_info_logstash file.
+      File.delete("#{provided_path}/.sfdc_info_logstash_ThisIsATestID00000")
+    end
+  end # Path for .sfdc_info_logstash
+end # describe SfdcElf

data/spec/spec_helper.rb

@@ -0,0 +1,38 @@
+# SimpleCov must be at the top of test class.
+require 'simplecov'
+SimpleCov.start
+
+require 'logstash/devutils/rspec/spec_helper'
+require 'lib/logstash/inputs/sfdc_elf'
+require 'webmock/rspec'
+require 'timecop'
+
+
+
+# Set up color and formatting for Rspec tests.
+RSpec.configure do |config|
+  # Use color in STDOUT
+  config.color = true
+
+  # Use color not only in STDOUT but also in pagers and files
+  config.tty = true
+
+  # Use the specified formatter
+  config.formatter = :documentation # :progress, :html, :textmate
+end
+
+
+
+# Turn off all connection to the internet.
+WebMock.disable_net_connect!(allow_localhost: true)
+
+
+
+# Helper methods to make it simple getting fixture data.
+def fixture_path
+  File.expand_path('../fixtures', __FILE__)
+end
+
+def fixture(file)
+  File.new(fixture_path + '/' + file)
+end

metadata

@@ -0,0 +1,180 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-sfdc_elf
+version: !ruby/object:Gem::Version
+  version: 1.0.2
+platform: ruby
+authors:
+- Abhishek Sreenivasa
+- Mohammed Islam
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: databasedotcom
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |-
+  This gem is a logstash plugin required to be installed on top of the Logstash core pipeline
+                         using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email:
+- asreenivasa@salesforce.com
+- mislam@salesforce.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- Gemfile
+- LICENSE.md
+- README.md
+- Rakefile
+- lib/logstash/inputs/sfdc_elf.rb
+- lib/logstash/inputs/sfdc_elf/client_with_streaming_support.rb
+- lib/logstash/inputs/sfdc_elf/queue_util.rb
+- lib/logstash/inputs/sfdc_elf/scheduler.rb
+- lib/logstash/inputs/sfdc_elf/state_persistor.rb
+- logstash-input-sfdc_elf.gemspec
+- spec/fixtures/auth_success_response.json
+- spec/fixtures/describe.json
+- spec/fixtures/org_query_response.json
+- spec/fixtures/queue_util/create_event_ELF_list1.json
+- spec/fixtures/queue_util/create_event_ELF_list2.json
+- spec/fixtures/queue_util/create_event_ELF_list3.json
+- spec/fixtures/queue_util/create_event_sampledata1.csv
+- spec/fixtures/queue_util/create_event_sampledata2.csv
+- spec/fixtures/queue_util/create_event_sampledata3.csv
+- spec/fixtures/queue_util/eventlogfile_describe.json
+- spec/fixtures/queue_util/eventlogfile_list.json
+- spec/fixtures/queue_util/sample_data1.csv
+- spec/fixtures/queue_util/sample_data2.csv
+- spec/fixtures/queue_util/sample_data3.csv
+- spec/fixtures/queue_util/sample_data4.csv
+- spec/fixtures/queue_util/sample_data5.csv
+- spec/inputs/sfdc_elf/queue_util_spec.rb
+- spec/inputs/sfdc_elf/scheduler_spec.rb
+- spec/inputs/sfdc_elf/state_persistor_spec.rb
+- spec/inputs/sfdc_elf_spec.rb
+- spec/spec_helper.rb
+homepage: http://www.salesforce.com
+licenses:
+- Salesforce EULA
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: A Logstash plugin the receives events from Salesforce EventLogFile
+test_files:
+- spec/fixtures/auth_success_response.json
+- spec/fixtures/describe.json
+- spec/fixtures/org_query_response.json
+- spec/fixtures/queue_util/create_event_ELF_list1.json
+- spec/fixtures/queue_util/create_event_ELF_list2.json
+- spec/fixtures/queue_util/create_event_ELF_list3.json
+- spec/fixtures/queue_util/create_event_sampledata1.csv
+- spec/fixtures/queue_util/create_event_sampledata2.csv
+- spec/fixtures/queue_util/create_event_sampledata3.csv
+- spec/fixtures/queue_util/eventlogfile_describe.json
+- spec/fixtures/queue_util/eventlogfile_list.json
+- spec/fixtures/queue_util/sample_data1.csv
+- spec/fixtures/queue_util/sample_data2.csv
+- spec/fixtures/queue_util/sample_data3.csv
+- spec/fixtures/queue_util/sample_data4.csv
+- spec/fixtures/queue_util/sample_data5.csv
+- spec/inputs/sfdc_elf/queue_util_spec.rb
+- spec/inputs/sfdc_elf/scheduler_spec.rb
+- spec/inputs/sfdc_elf/state_persistor_spec.rb
+- spec/inputs/sfdc_elf_spec.rb
+- spec/spec_helper.rb