logstash-input-sfdc_elf 1.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 156b5331cb015cdb70ac95c8dd4310d5eb890826
+  data.tar.gz: 0400cb3ec9a08acc72954d744c721c7dacfd1b1b
+SHA512:
+  metadata.gz: e816e475d111272dc91011dac75ea5579098dbe0ead14a04e02e4b9b423ac58531fa285b8698f4b91cf95e70e8a8a43e99a81a3bf74223c05254e3a6b3df13a9
+  data.tar.gz: ac8df1947c0fff174c489e1a2c94de2f247184d1318ea1a48dd498ad440d21354081e11b2bccdc6964a96d3e1a0b6e8ab6e1abcc793073efd4764f96eaa27465

data/.gitignore

@@ -0,0 +1,11 @@
+*.gem
+Gemfile.lock
+.bundle
+vendor
+*.iml
+.idea
+coverage
+.DS_Store
+lib/.DS_Store
+lib/logstash/.DS_Store
+lib/logstash/inputs/.DS_Store

data/.rubocop.yml

@@ -0,0 +1,41 @@
+# Must be compact, so that Logstash can locate plugins via "LogStash::inputs::YourPluginName"
+Style/ClassAndModuleChildren:
+  EnforcedStyle: compact
+
+# ABC is a metric based on the number of Assignments, Branches, and Conditions.
+# More info: http://c2.com/cgi/wiki?AbcMetric
+AbcSize:
+  Max: 25
+
+# Disable messages about using fail instead of raise for a specific try/catch block.
+SignalException:
+  Enabled: false
+
+# Disable message about providing an exception class and message as an argument to raise.
+RaiseArgs:
+  EnforcedStyle: compact
+
+Metrics/LineLength:
+  AllowURI: true
+  Max: 120
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/ClassLength:
+  Max: 150
+
+EmptyLines:
+  Enabled: false
+
+# Allow to have no line break between method and modifier. Example using public or private, then method.
+Style/EmptyLinesAroundAccessModifier:
+  Enabled: false
+
+# Allow to move chained dot methods to the next line.
+Style/DotPosition:
+  Enabled: false
+
+# Allow to use the word "get" in accessor method names.
+Style/AccessorMethodName:
+  Enabled: false

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+gemspec
+
+group :test do
+  gem 'rubocop', '>= 0.27'
+  gem 'simplecov', '>= 0.9'
+  gem 'webmock'
+  gem 'timecop'
+end

data/LICENSE.md

@@ -0,0 +1,12 @@
+# Logstash plugin for EventLogFile End User License Agreement
+
+Except as described below, Logstash plugin for EventLogFile is Copyright (c) 2015, salesforce.com, inc. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+
+* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+* Neither the name of salesforce.com, inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1 @@
+# Logstash Plugin for Salesforce's Event Log File

data/Rakefile

@@ -0,0 +1,13 @@
+@files = []
+
+require 'rubocop/rake_task'
+RuboCop::RakeTask.new
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+require 'logstash/devutils/rake'
+
+task test: :spec
+
+task default: :test

data/lib/logstash/inputs/sfdc_elf.rb

@@ -0,0 +1,147 @@
+# encoding: utf-8
+require 'logstash/inputs/base'
+require 'logstash/namespace'
+require_relative 'sfdc_elf/client_with_streaming_support'
+require_relative 'sfdc_elf/queue_util'
+require_relative 'sfdc_elf/state_persistor'
+require_relative 'sfdc_elf/scheduler'
+
+# This plugin enables Salesforce customers to load EventLogFile(ELF) data from their Force.com orgs. The plugin will
+# handle downloading ELF CSV file, parsing them, and handling any schema changes transparently.
+class LogStash::Inputs::SfdcElf < LogStash::Inputs::Base
+  LOG_KEY        = 'SFDC'
+  RETRY_ATTEMPTS = 3
+
+  config_name 'sfdc_elf'
+  default :codec, 'plain'
+
+  # Username to your Force.com organization.
+  config :username, validate: :string, required: true
+
+  # Password to your Force.com organization.
+  config :password, validate: :password, required: true
+
+  # Client id to your Force.com organization.
+  config :client_id, validate: :password, required: true
+
+  # Client secret to your Force.com organization.
+  config :client_secret, validate: :password, required: true
+
+  # The host to use for OAuth2 authentication.
+  config :host, validate: :string, default: 'login.salesforce.com'
+
+  # Only needed when your Force.com organization requires it.
+  # Security token to you Force.com organization, can be found in  My Settings > Personal > Reset My Security Token.
+  # Then it will take you to "Reset My Security Token" page, and click on the "Reset Security Token" button. The token
+  # will be emailed to you.
+  config :security_token, validate: :password, default: ''
+
+  # The path to be use to store the .sfdc_info_logstash state persistor file. You set the path like so, `~/SomeDirectory` Paths must be
+  # absolute and cannot be relative.
+  config :path, validate: :string, default: Dir.home
+
+  # Specify how often the plugin should grab new data in terms of minutes.
+  config :poll_interval_in_minutes, validate: [*1..(24 * 60)], default: (24 * 60)
+
+
+  # The first part of logstash pipeline is register, where all instance variables are initialized.
+
+  public
+  def register
+    # Initialize the client.
+    @client = ClientWithStreamingSupport.new
+    @client.client_id     = @client_id.value
+    @client.client_secret = @client_secret.value
+    @client.host          = @host
+    @client.version       = '33.0'
+
+    # Authenticate the client
+    @logger.info("#{LOG_KEY}: tyring to authenticate client")
+    @client.retryable_authenticate(username: @username,
+                                   password: @password.value + @security_token.value,
+                                   retry_attempts: RETRY_ATTEMPTS)
+    @logger.info("#{LOG_KEY}: authenticating succeeded")
+
+    # Save org id to distinguish between multiple orgs.
+    @org_id = @client.query('select id from Organization')[0]['Id']
+
+    # Set up time interval for forever while loop.
+    @poll_interval_in_seconds = @poll_interval_in_minutes * 60
+
+    # Handel the @path config passed by the user. If path does not exist then set @path to home directory.
+    verify_path
+
+    # Handel parsing the data into event objects and enqueue it to the queue.
+    @queue_util = QueueUtil.new
+
+    # Handel when to schedule the next process based on the @poll_interval_in_hours config.
+    @scheduler = Scheduler.new(@poll_interval_in_seconds)
+
+    # Handel state of the plugin based on the read and writes of LogDates to the .sdfc_info_logstash file.
+    @state_persistor = StatePersistor.new(@path, @org_id)
+
+    # Grab the last indexed log date.
+    @last_indexed_log_date = @state_persistor.get_last_indexed_log_date
+    @logger.info("#{LOG_KEY}: @last_indexed_log_date =  #{@last_indexed_log_date}")
+  end  # def register
+
+
+
+
+  # The second stage of Logstash pipeline is run, where it expects to parse your data into event objects and then pass
+  # it into the queue to be used in the rest of the pipeline.
+
+  public
+  def run(queue)
+    @scheduler.schedule do
+      # Line for readable log statements.
+      @logger.info('---------------------------------------------------')
+
+      # Grab a list of SObjects, specifically EventLogFiles.
+      soql_expr = "SELECT Id, EventType, Logfile, LogDate, LogFileLength, LogFileFieldTypes
+                   FROM EventLogFile
+                   WHERE LogDate > #{@last_indexed_log_date} ORDER BY LogDate ASC "
+
+
+      query_result_list = @client.retryable_query(username: @username,
+                                                  password: @password.value + @security_token.value,
+                                                  retry_attempts: RETRY_ATTEMPTS,
+                                                  soql_expr: soql_expr)
+
+      @logger.info("#{LOG_KEY}: query result size = #{query_result_list.size}")
+
+      if !query_result_list.empty?
+        # query_result_list is in ascending order based on the LogDate, so grab the last one of the list and save the
+        # LogDate to @last_read_log_date and .sfdc_info_logstash
+        @last_indexed_log_date = query_result_list.last.LogDate.strftime('%FT%T.%LZ')
+
+        # TODO: grab tempfiles here!!
+
+        # Overwrite the .sfdc_info_logstash file with the @last_read_log_date.
+        # Note: we currently do not support deduplication, but will implement it soon.
+        # TODO: need to implement deduplication
+        # TODO: might have to move this after enqueue_events(), in case of a crash in between.
+        # TODO: can do all @state_persistor calls after the if statement
+        @state_persistor.update_last_indexed_log_date(@last_indexed_log_date)
+
+        # Creates events from query_result_list, then simply append the events to the queue.
+        @queue_util.enqueue_events(query_result_list, queue, @client)
+      end
+    end # do loop
+  end # def run
+
+
+
+
+  # Handel the @path variable passed by the user. If path does not exist then set @path to home directory.
+
+  private
+  def verify_path
+    # Check if the path exist, if not then set @path to home directory.
+    unless File.directory?(@path)
+      @logger.warn("#{LOG_KEY}: provided path does not exist or is invalid. path=#{@path}")
+      @path = Dir.home
+    end
+    @logger.info("#{LOG_KEY}: path = #{@path}")
+  end
+end # class LogStash::inputs::File

data/lib/logstash/inputs/sfdc_elf/client_with_streaming_support.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+require 'databasedotcom'
+
+# This class subclasses Databasedotcom Client object and added steaming
+# downloading and retryable authentication and retryable query.
+class ClientWithStreamingSupport < Databasedotcom::Client
+  # Constants
+  # LOG_KEY        = 'SFDC - ClientWithStreamingSupport'
+  #
+  # def initialize
+  #   @logger = Cabin::Channel.get(LogStash)
+  # end
+
+  def streaming_download(path, output_stream)
+    connection = Net::HTTP.new(URI.parse(instance_url).host, 443)
+    connection.use_ssl = true
+    encoded_path = URI.escape(path)
+
+    req = Net::HTTP::Get.new(encoded_path, 'Authorization' => "OAuth #{oauth_token}")
+    connection.request(req) do |response|
+      raise SalesForceError.new(response) unless response.is_a?(Net::HTTPSuccess)
+      response.read_body do |chunk|
+        output_stream.write chunk
+      end
+    end
+  end
+
+  # This helper method is called whenever we need to initaialize the client
+  # object or whenever the client token expires. It will attempt 3 times
+  # with a 30 second delay between each retry. On the 3th try, if it fails the
+  # exception will be raised.
+
+  def retryable_authenticate(options = {})
+    1.upto(options[:retry_attempts]) do |count|
+      begin
+        # If exception is not thrown, then break out of loop.
+        authenticate(username: options[:username], password: options[:password])
+        break
+      rescue StandardError => e
+        # Sleep for 30 seconds 2 times. On the 3th time if it fails raise the exception without sleeping.
+        if (count == options[:retry_attempts])
+          raise e
+        else
+          sleep(30)
+        end
+      end
+    end
+  end # def authenticate
+
+
+  def retryable_query(options = {})
+    query(options[:soql_expr])
+  rescue Databasedotcom::SalesForceError => e
+    # Session has expired. Force user logout, then re-authenticate.
+    if e.message == 'Session expired or invalid'
+      retryable_authenticate(options)
+    else
+      raise e
+    end
+  end # def retryable_query
+end # ClientWithStreamingSupport

data/lib/logstash/inputs/sfdc_elf/queue_util.rb

@@ -0,0 +1,176 @@
+# encoding: utf-8
+require 'csv'
+require 'resolv'
+
+# Handel parsing data into event objects and then enqueue all of the events to the queue.
+class QueueUtil
+  # Constants
+  LOG_KEY        = 'SFDC - QueueUtil'
+  SEPARATOR      = ','
+  QUOTE_CHAR     = '"'
+
+  # Zip up the tempfile, which is a CSV file, and the field types, so that when parsing the CSV file we can accurately
+  # convert each field to its respective type. Like Integers and Booleans.
+  EventLogFile = Struct.new(:field_types, :temp_file, :event_type)
+
+
+  def initialize
+    @logger = Cabin::Channel.get(LogStash)
+  end
+
+
+
+
+  # Given a list of query result's, iterate through it and grab the CSV file associated with it. Then parse the CSV file
+  # line by line and generating the event object for it. Then enqueue it.
+
+  public
+  def enqueue_events(query_result_list, queue, client)
+    @logger.info("#{LOG_KEY}: enqueue events")
+
+    # Grab a list of Tempfiles that contains CSV file data.
+    event_log_file_records = get_event_log_file_records(query_result_list, client)
+
+    # Iterate though each record.
+    event_log_file_records.each do |elf|
+      begin
+        # Create local variable to simplify & make code more readable.
+        tmp = elf.temp_file
+
+        # Get the schema from the first line in the tempfile. It will be in CSV format so we parse it, and it will
+        # return an array.
+        schema = CSV.parse_line(tmp.readline, col_sep: SEPARATOR, quote_char: QUOTE_CHAR)
+
+        # Loop through tempfile, line by line.
+        tmp.each_line do |line|
+          # Parse the current line, it will return an string array.
+          string_array = CSV.parse_line(line, col_sep: SEPARATOR, quote_char: QUOTE_CHAR)
+
+          # Convert the string array into its corresponding type array.
+          data = string_to_type_array(string_array, elf.field_types)
+
+          # create_event will return a event object.
+          queue << create_event(schema, data, elf.event_type)
+        end
+      ensure
+        # Close tmp file and unlink it, doing this will delete the actual tempfile.
+        tmp.close
+        tmp.unlink
+      end
+    end # do loop, tempfile_list
+  end # def create_event_list
+
+
+
+
+  # Convert the given string array to its corresponding type array and return it.
+
+  private
+  def string_to_type_array(string_array, field_types)
+    data = []
+
+    field_types.each_with_index do |type, i|
+      case type
+        when 'Number'
+          data[i] = (string_array[i].empty?) ? nil : string_array[i].to_f
+        when 'Boolean'
+          data[i] = (string_array[i].empty?) ? nil : (string_array[i] == '0')
+        when 'IP'
+          data[i] = valid_ip(string_array[i]) ? string_array[i] : nil
+        else # 'String', 'Id', 'EscapedString', 'Set'
+          data[i] = (string_array[i].empty?) ? nil : string_array[i]
+      end
+    end # do loop
+
+    data
+  end # convert_string_to_type
+
+
+
+  # Check if the given ip address is truely an ip address.
+
+  private
+  def valid_ip(ip)
+    ip =~ Resolv::IPv4::Regex ? true : false
+  end
+
+
+
+
+  # Bases on the schema and data, we create the event object. At any point if the data is nil we simply dont add
+  # the data to the event object. Special handling is needed when the schema 'TIMESTAMP' occurs, then the data
+  # associated with it needs to be converted into a LogStash::Timestamp.
+
+  private
+  def create_event(schema, data, event_type)
+    # Initaialize event to be used. @timestamp and @version is automatically added
+    event = LogStash::Event.new
+
+    # Add column data pair to event.
+    data.each_index do |i|
+      # Grab current key.
+      schema_name = schema[i]
+
+      # Handle when field_name is 'TIMESTAMP', Change the @timestamp field to the actual time on the CSV file,
+      # but convert it to iso8601.
+      if schema_name == 'TIMESTAMP'
+        epochmillis = DateTime.parse(data[i]).to_time.to_f
+        event.timestamp = LogStash::Timestamp.at(epochmillis)
+      end
+
+      # Allow Elasticsearch index's have to types set to EventType.
+      event['type'] = event_type.downcase
+
+      # Add the schema data pair to event object.
+      if data[i] != nil
+        event[schema_name] = data[i]
+      end
+    end
+
+    # Return the event
+    event
+  end # def create_event
+
+
+
+
+  # This helper method takes as input a list/collection of SObjects which each contains a path to their respective CSV
+  # files. The path is stored in the LogFile field. Using that path, we are able to grab the actual CSV file via
+  # @client.http_get method.
+  #
+  # After grabbing the CSV file we then store them using the standard Tempfile library. Tempfile will create a unique
+  # file each time using 'sfdc_elf_tempfile' as the prefix and finally we will be returning a list of Tempfile object,
+  # where the user can read the Tempfile and then close it and unlink it, which will delete the file.
+
+  public
+  def get_event_log_file_records(query_result_list, client)
+    @logger.info("#{LOG_KEY}: generating tempfile list")
+    result = []
+    query_result_list.each do |event_log_file|
+      # Get the path of the CSV file from the LogFile field, then stream the data to the .write method of the Tempfile
+      tmp = Tempfile.new('sfdc_elf_tempfile')
+      client.streaming_download(event_log_file.LogFile, tmp)
+
+      # Flushing will write the buffer into the Tempfile itself.
+      tmp.flush
+
+      # Rewind will move the file pointer from the end to the beginning of the file, so that users can simple
+      # call the Read method.
+      tmp.rewind
+
+      # Append the EventLogFile object into the result list
+      field_types = event_log_file.LogFileFieldTypes.split(',')
+      result << EventLogFile.new(field_types, tmp, event_log_file.EventType)
+
+      # Log the info from event_log_file object.
+      @logger.info("  #{LOG_KEY}: Id = #{event_log_file.Id}")
+      @logger.info("  #{LOG_KEY}: EventType = #{event_log_file.EventType}")
+      @logger.info("  #{LOG_KEY}: LogFile = #{event_log_file.LogFile}")
+      @logger.info("  #{LOG_KEY}: LogDate = #{event_log_file.LogDate}")
+      @logger.info("  #{LOG_KEY}: LogFileLength = #{event_log_file.LogFileLength}")
+      @logger.info("  #{LOG_KEY}: LogFileFieldTypes = #{event_log_file.LogFileFieldTypes}")
+      @logger.info('  ......................................')
+    end
+    result
+  end # def get_event_log_file_records
+end # QueueUtil