logstash-input-sfdc_elf_schneider 1.0.5

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 152dbc61c42b31c6dae7a176404074ca2614d8717006d5d7c101a4b7b45efdd7
+  data.tar.gz: d1d649228b720404964e7cccc34c49ce409e7e2dd786adbfdc7376972f270003
+SHA512:
+  metadata.gz: 4fe45fe63f01fe4d93d1f42383613c55372a88d668a05bad355ec37c253f011182608580bc0d8079af50ca22aa286f6bde543d2d75f535e7151af3a247f891f1
+  data.tar.gz: c56974381aae6b844f113b5f417f8e427c1c2e34bde92bad52a80ddf52dc2e7c1f9d7f629d23cd7b398b76f5ca851ba3ad20978b04d05f9d3fbdf193608aa19e

data/.gitignore

@@ -0,0 +1,11 @@
+*.gem
+Gemfile.lock
+.bundle
+vendor
+*.iml
+.idea
+coverage
+.DS_Store
+lib/.DS_Store
+lib/logstash/.DS_Store
+lib/logstash/inputs/.DS_Store

data/.rubocop.yml

@@ -0,0 +1,41 @@
+# Must be compact, so that Logstash can locate plugins via "LogStash::inputs::YourPluginName"
+Style/ClassAndModuleChildren:
+  EnforcedStyle: compact
+
+# ABC is a metric based on the number of Assignments, Branches, and Conditions.
+# More info: http://c2.com/cgi/wiki?AbcMetric
+AbcSize:
+  Max: 25
+
+# Disable messages about using fail instead of raise for a specific try/catch block.
+SignalException:
+  Enabled: false
+
+# Disable message about providing an exception class and message as an argument to raise.
+RaiseArgs:
+  EnforcedStyle: compact
+
+Metrics/LineLength:
+  AllowURI: true
+  Max: 120
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/ClassLength:
+  Max: 150
+
+EmptyLines:
+  Enabled: false
+
+# Allow to have no line break between method and modifier. Example using public or private, then method.
+Style/EmptyLinesAroundAccessModifier:
+  Enabled: false
+
+# Allow to move chained dot methods to the next line.
+Style/DotPosition:
+  Enabled: false
+
+# Allow to use the word "get" in accessor method names.
+Style/AccessorMethodName:
+  Enabled: false

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+gemspec
+
+group :test do
+  gem 'rubocop', '>= 0.27'
+  gem 'simplecov', '>= 0.9'
+  gem 'webmock'
+  gem 'timecop'
+end

data/MIT-LICENSE

@@ -0,0 +1,7 @@
+Copyright 2015 salesforce.com, inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,65 @@
+# Logtypes support
+
+we only support 'ApexExecution','ApexSoap','ApexTrigger','API','BulkApi','Dashboard','LightningError','LightningInteraction','LightningPageView','LightningPerformance','LoginAs','Login','Logout','MetadataApiOperation','Report','RestApi','URI','VisualforceRequest','WaveChange','WaveInteraction','WavePerformance' log types.
+
+# Logstash Plugin for Salesforce's Event Log File
+
+The Salesforce Event Log File [Ruby Gem](https://rubygems.org/gems/logstash-input-sfdc_elf/) plug-in simplifies the integration between
+the ELK stack and Event Log Files by allowing you to easily download and index Salesforce event data every day without a custom integration.
+
+ELF on ELK on Docker is the quickest way to get started. Check the links below.
+
+### Logstash Plugin Config
+
+```
+input {
+    sfdc_elf {
+        # === Required Fields ===
+
+        # Input the username and password of your Force.com organization.
+        username       => ""
+        password       => ""
+
+        # Id & Secret are values that are generated when you create a "Connected App."
+        client_id      => ""
+        client_secret  => ""
+
+
+        # === OPTIONAL FIELDS ===
+        # Leave these fields commented out (with '#') if the config does not apply to you. Some of the fields
+        # have default values
+
+        # Only needed when your Force.com organization requires it.
+        # security_token => ""
+
+        # The path to be use to store the .sfdc_info_logstash state persistor file. You set the path
+        # like so, "~/SomeDirectory" Paths must be absolute and cannot be relative.
+        # Defaults to your home directory "~/".
+        # path           => ""
+
+        # Specify how often the plugin should grab new data in terms of minutes.
+        # Defaults to 1440 minutes (1 day). We recommend keeping it at 1 day.
+        # poll_interval_in_minutes => 60
+
+        # The host to use for OAuth2 authentication.
+        # Defaluts to "login.salesforce.com".
+        # Use "test.salesforce.com" for connecting to Sandbox instance.
+        # host           => ""
+
+    }
+}
+```
+
+### Setting up a Salesforce Connected App
+
+Detailed instructions for setting up a Connected App can be found [here](https://help.salesforce.com/apex/HTViewHelpDoc?id=connected_app_create.htm).
+When configuring the connected application, ensure the following options are configured:
+
+1. _Enable OAuth Settings_ is checked.
+2. _Access and manage data (api)_ and _Access your basic information (id, profile, email, address, phone)_ are included in your _Selected OAuth Scopes_.
+
+### Blogs, Articles, and Tutorials
+
+1. [Elf on Elk on Docker](http://www.salesforcehacker.com/2015/10/elf-on-elk-on-docker.html) by Adam Torman
+2. [Elf on Elk on Docker Image Source Code](https://github.com/developerforce/elf_elk_docker/blob/master/README.md)
+3. ['Users: WE KNOW THEM' – The ELF@Salesforce](https://www.elastic.co/elasticon/conf/2016/sf/users-we-know-them-the-elf-at-salesforce) at Elastic{ON} '16 by Adam Torman and Abhishek Sreenivasa

data/Rakefile

@@ -0,0 +1,13 @@
+@files = []
+
+require 'rubocop/rake_task'
+RuboCop::RakeTask.new
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+require 'logstash/devutils/rake'
+
+task test: :spec
+
+task default: :test

data/lib/logstash/inputs/sfdc_elf.rb

@@ -0,0 +1,147 @@
+# encoding: utf-8
+require 'logstash/inputs/base'
+require 'logstash/namespace'
+require_relative 'sfdc_elf/client_with_streaming_support'
+require_relative 'sfdc_elf/queue_util'
+require_relative 'sfdc_elf/state_persistor'
+require_relative 'sfdc_elf/scheduler'
+
+# This plugin enables Salesforce customers to load EventLogFile(ELF) data from their Force.com orgs. The plugin will
+# handle downloading ELF CSV file, parsing them, and handling any schema changes transparently.
+class LogStash::Inputs::SfdcElf < LogStash::Inputs::Base
+  LOG_KEY        = 'SFDC'
+  RETRY_ATTEMPTS = 3
+
+  config_name 'sfdc_elf'
+  default :codec, 'plain'
+
+  # Username to your Force.com organization.
+  config :username, validate: :string, required: true
+
+  # Password to your Force.com organization.
+  config :password, validate: :password, required: true
+
+  # Client id to your Force.com organization.
+  config :client_id, validate: :password, required: true
+
+  # Client secret to your Force.com organization.
+  config :client_secret, validate: :password, required: true
+
+  # The host to use for OAuth2 authentication.
+  config :host, validate: :string, default: 'login.salesforce.com'
+
+  # Only needed when your Force.com organization requires it.
+  # Security token to you Force.com organization, can be found in  My Settings > Personal > Reset My Security Token.
+  # Then it will take you to "Reset My Security Token" page, and click on the "Reset Security Token" button. The token
+  # will be emailed to you.
+  config :security_token, validate: :password, default: ''
+
+  # The path to be use to store the .sfdc_info_logstash state persistor file. You set the path like so, `~/SomeDirectory` Paths must be
+  # absolute and cannot be relative.
+  config :path, validate: :string, default: Dir.home
+
+  # Specify how often the plugin should grab new data in terms of minutes.
+  config :poll_interval_in_minutes, validate: [*1..(24 * 60)], default: (24 * 60)
+
+
+  # The first part of logstash pipeline is register, where all instance variables are initialized.
+
+  public
+  def register
+    # Initialize the client.
+    @client = ClientWithStreamingSupport.new
+    @client.client_id     = @client_id.value
+    @client.client_secret = @client_secret.value
+    @client.host          = @host
+    @client.version       = '33.0'
+
+    # Authenticate the client
+    @logger.info("#{LOG_KEY}: tyring to authenticate client")
+    @client.retryable_authenticate(username: @username,
+                                   password: @password.value + @security_token.value,
+                                   retry_attempts: RETRY_ATTEMPTS)
+    @logger.info("#{LOG_KEY}: authenticating succeeded")
+
+    # Save org id to distinguish between multiple orgs.
+    @org_id = @client.query('select id from Organization')[0]['Id']
+
+    # Set up time interval for forever while loop.
+    @poll_interval_in_seconds = @poll_interval_in_minutes * 60
+
+    # Handel the @path config passed by the user. If path does not exist then set @path to home directory.
+    verify_path
+
+    # Handel parsing the data into event objects and enqueue it to the queue.
+    @queue_util = QueueUtil.new
+
+    # Handel when to schedule the next process based on the @poll_interval_in_hours config.
+    @scheduler = Scheduler.new(@poll_interval_in_seconds)
+
+    # Handel state of the plugin based on the read and writes of LogDates to the .sdfc_info_logstash file.
+    @state_persistor = StatePersistor.new(@path, @org_id)
+
+    # Grab the last indexed log date.
+    @last_indexed_log_date = @state_persistor.get_last_indexed_log_date
+    @logger.info("#{LOG_KEY}: @last_indexed_log_date =  #{@last_indexed_log_date}")
+  end  # def register
+
+
+
+
+  # The second stage of Logstash pipeline is run, where it expects to parse your data into event objects and then pass
+  # it into the queue to be used in the rest of the pipeline.
+
+  public
+  def run(queue)
+    @scheduler.schedule do
+      # Line for readable log statements.
+      @logger.info('---------------------------------------------------')
+
+      # Grab a list of SObjects, specifically EventLogFiles.
+      soql_expr = "SELECT Id, EventType, Logfile, LogDate, LogFileLength, LogFileFieldTypes
+                   FROM EventLogFile
+                   WHERE LogDate > #{@last_indexed_log_date} and EventType in ('ApexExecution','ApexSoap','ApexTrigger','API','BulkApi','Dashboard','LightningError','LightningInteraction','LightningPageView','LightningPerformance','LoginAs','Login','Logout','MetadataApiOperation','Report','RestApi','URI','VisualforceRequest','WaveChange','WaveInteraction','WavePerformance') ORDER BY LogDate ASC "
+
+
+      query_result_list = @client.retryable_query(username: @username,
+                                                  password: @password.value + @security_token.value,
+                                                  retry_attempts: RETRY_ATTEMPTS,
+                                                  soql_expr: soql_expr)
+
+      @logger.info("#{LOG_KEY}: query result size = #{query_result_list.size}")
+
+      if !query_result_list.empty?
+        # query_result_list is in ascending order based on the LogDate, so grab the last one of the list and save the
+        # LogDate to @last_read_log_date and .sfdc_info_logstash
+        @last_indexed_log_date = query_result_list.last.LogDate.strftime('%FT%T.%LZ')
+
+        # TODO: grab tempfiles here!!
+
+        # Overwrite the .sfdc_info_logstash file with the @last_read_log_date.
+        # Note: we currently do not support deduplication, but will implement it soon.
+        # TODO: need to implement deduplication
+        # TODO: might have to move this after enqueue_events(), in case of a crash in between.
+        # TODO: can do all @state_persistor calls after the if statement
+        @state_persistor.update_last_indexed_log_date(@last_indexed_log_date)
+
+        # Creates events from query_result_list, then simply append the events to the queue.
+        @queue_util.enqueue_events(query_result_list, queue, @client)
+      end
+    end # do loop
+  end # def run
+
+
+
+
+  # Handel the @path variable passed by the user. If path does not exist then set @path to home directory.
+
+  private
+  def verify_path
+    # Check if the path exist, if not then set @path to home directory.
+    unless File.directory?(@path)
+      @logger.warn("#{LOG_KEY}: provided path does not exist or is invalid. path=#{@path}")
+      @path = Dir.home
+    end
+    @logger.info("#{LOG_KEY}: path = #{@path}")
+  end
+end # class LogStash::inputs::File

data/lib/logstash/inputs/sfdc_elf/client_with_streaming_support.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+require 'databasedotcom'
+
+# This class subclasses Databasedotcom Client object and added steaming
+# downloading and retryable authentication and retryable query.
+class ClientWithStreamingSupport < Databasedotcom::Client
+  # Constants
+  # LOG_KEY        = 'SFDC - ClientWithStreamingSupport'
+  #
+  # def initialize
+  #   @logger = Cabin::Channel.get(LogStash)
+  # end
+
+  def streaming_download(path, output_stream)
+    connection = Net::HTTP.new(URI.parse(instance_url).host, 443)
+    connection.use_ssl = true
+    encoded_path = URI.escape(path)
+
+    req = Net::HTTP::Get.new(encoded_path, 'Authorization' => "OAuth #{oauth_token}")
+    connection.request(req) do |response|
+      raise SalesForceError.new(response) unless response.is_a?(Net::HTTPSuccess)
+      response.read_body do |chunk|
+        output_stream.write chunk
+      end
+    end
+  end
+
+  # This helper method is called whenever we need to initaialize the client
+  # object or whenever the client token expires. It will attempt 3 times
+  # with a 30 second delay between each retry. On the 3th try, if it fails the
+  # exception will be raised.
+
+  def retryable_authenticate(options = {})
+    1.upto(options[:retry_attempts]) do |count|
+      begin
+        # If exception is not thrown, then break out of loop.
+        authenticate(username: options[:username], password: options[:password])
+        break
+      rescue StandardError => e
+        # Sleep for 30 seconds 2 times. On the 3th time if it fails raise the exception without sleeping.
+        if (count == options[:retry_attempts])
+          raise e
+        else
+          sleep(30)
+        end
+      end
+    end
+  end # def authenticate
+
+
+  def retryable_query(options = {})
+    query(options[:soql_expr])
+  rescue Databasedotcom::SalesForceError => e
+    # Session has expired. Force user logout, then re-authenticate.
+    if e.message == 'Session expired or invalid'
+      retryable_authenticate(options)
+    else
+      raise e
+    end
+  end # def retryable_query
+end # ClientWithStreamingSupport

data/lib/logstash/inputs/sfdc_elf/queue_util.rb

@@ -0,0 +1,176 @@
+# encoding: utf-8
+require 'csv'
+require 'resolv'
+
+# Handel parsing data into event objects and then enqueue all of the events to the queue.
+class QueueUtil
+  # Constants
+  LOG_KEY        = 'SFDC - QueueUtil'
+  SEPARATOR      = ','
+  QUOTE_CHAR     = '"'
+
+  # Zip up the tempfile, which is a CSV file, and the field types, so that when parsing the CSV file we can accurately
+  # convert each field to its respective type. Like Integers and Booleans.
+  EventLogFile = Struct.new(:field_types, :temp_file, :event_type)
+
+
+  def initialize
+    @logger = Cabin::Channel.get(LogStash)
+  end
+
+
+
+
+  # Given a list of query result's, iterate through it and grab the CSV file associated with it. Then parse the CSV file
+  # line by line and generating the event object for it. Then enqueue it.
+
+  public
+  def enqueue_events(query_result_list, queue, client)
+    @logger.info("#{LOG_KEY}: enqueue events")
+
+    # Grab a list of Tempfiles that contains CSV file data.
+    event_log_file_records = get_event_log_file_records(query_result_list, client)
+
+    # Iterate though each record.
+    event_log_file_records.each do |elf|
+      begin
+        # Create local variable to simplify & make code more readable.
+        tmp = elf.temp_file
+
+        # Get the schema from the first line in the tempfile. It will be in CSV format so we parse it, and it will
+        # return an array.
+        schema = CSV.parse_line(tmp.readline, col_sep: SEPARATOR, quote_char: QUOTE_CHAR)
+
+        # Loop through tempfile, line by line.
+        tmp.each_line do |line|
+          # Parse the current line, it will return an string array.
+          string_array = CSV.parse_line(line, col_sep: SEPARATOR, quote_char: QUOTE_CHAR)
+
+          # Convert the string array into its corresponding type array.
+          data = string_to_type_array(string_array, elf.field_types)
+
+          # create_event will return a event object.
+          queue << create_event(schema, data, elf.event_type)
+        end
+      ensure
+        # Close tmp file and unlink it, doing this will delete the actual tempfile.
+        tmp.close
+        tmp.unlink
+      end
+    end # do loop, tempfile_list
+  end # def create_event_list
+
+
+
+
+  # Convert the given string array to its corresponding type array and return it.
+
+  private
+  def string_to_type_array(string_array, field_types)
+    data = []
+
+    field_types.each_with_index do |type, i|
+      case type
+        when 'Number'
+          data[i] = (string_array[i].empty?) ? nil : string_array[i].to_f
+        when 'Boolean'
+          data[i] = (string_array[i].empty?) ? nil : (string_array[i] == '0')
+        when 'IP'
+          data[i] = valid_ip(string_array[i]) ? string_array[i] : nil
+        else # 'String', 'Id', 'EscapedString', 'Set'
+          data[i] = (string_array[i].empty?) ? nil : string_array[i]
+      end
+    end # do loop
+
+    data
+  end # convert_string_to_type
+
+
+
+  # Check if the given ip address is truely an ip address.
+
+  private
+  def valid_ip(ip)
+    ip =~ Resolv::IPv4::Regex ? true : false
+  end
+
+
+
+
+  # Bases on the schema and data, we create the event object. At any point if the data is nil we simply dont add
+  # the data to the event object. Special handling is needed when the schema 'TIMESTAMP' occurs, then the data
+  # associated with it needs to be converted into a LogStash::Timestamp.
+
+  private
+  def create_event(schema, data, event_type)
+    # Initaialize event to be used. @timestamp and @version is automatically added
+    event = LogStash::Event.new
+
+    # Add column data pair to event.
+    data.each_index do |i|
+      # Grab current key.
+      schema_name = schema[i]
+
+      # Handle when field_name is 'TIMESTAMP', Change the @timestamp field to the actual time on the CSV file,
+      # but convert it to iso8601.
+      if schema_name == 'TIMESTAMP'
+        epochmillis = DateTime.parse(data[i]).to_time.to_f
+        event.timestamp = LogStash::Timestamp.at(epochmillis)
+      end
+
+      # Allow Elasticsearch index's have to types set to EventType.
+      event['type'] = event_type.downcase
+
+      # Add the schema data pair to event object.
+      if data[i] != nil
+        event[schema_name] = data[i]
+      end
+    end
+
+    # Return the event
+    event
+  end # def create_event
+
+
+
+
+  # This helper method takes as input a list/collection of SObjects which each contains a path to their respective CSV
+  # files. The path is stored in the LogFile field. Using that path, we are able to grab the actual CSV file via
+  # @client.http_get method.
+  #
+  # After grabbing the CSV file we then store them using the standard Tempfile library. Tempfile will create a unique
+  # file each time using 'sfdc_elf_tempfile' as the prefix and finally we will be returning a list of Tempfile object,
+  # where the user can read the Tempfile and then close it and unlink it, which will delete the file.
+
+  public
+  def get_event_log_file_records(query_result_list, client)
+    @logger.info("#{LOG_KEY}: generating tempfile list")
+    result = []
+    query_result_list.each do |event_log_file|
+      # Get the path of the CSV file from the LogFile field, then stream the data to the .write method of the Tempfile
+      tmp = Tempfile.new('sfdc_elf_tempfile')
+      client.streaming_download(event_log_file.LogFile, tmp)
+
+      # Flushing will write the buffer into the Tempfile itself.
+      tmp.flush
+
+      # Rewind will move the file pointer from the end to the beginning of the file, so that users can simple
+      # call the Read method.
+      tmp.rewind
+
+      # Append the EventLogFile object into the result list
+      field_types = event_log_file.LogFileFieldTypes.split(',')
+      result << EventLogFile.new(field_types, tmp, event_log_file.EventType)
+
+      # Log the info from event_log_file object.
+      @logger.info("  #{LOG_KEY}: Id = #{event_log_file.Id}")
+      @logger.info("  #{LOG_KEY}: EventType = #{event_log_file.EventType}")
+      @logger.info("  #{LOG_KEY}: LogFile = #{event_log_file.LogFile}")
+      @logger.info("  #{LOG_KEY}: LogDate = #{event_log_file.LogDate}")
+      @logger.info("  #{LOG_KEY}: LogFileLength = #{event_log_file.LogFileLength}")
+      @logger.info("  #{LOG_KEY}: LogFileFieldTypes = #{event_log_file.LogFileFieldTypes}")
+      @logger.info('  ......................................')
+    end
+    result
+  end # def get_event_log_file_records
+end # QueueUtil