logstash-input-crowdstrike_fdr 2.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,53 @@
1
+ class MagicGzipValidator
2
+ attr_reader :file
3
+ attr_reader :starting_signature
4
+
5
+ VALID_STARTING_SIGNATURE = "1f8b"
6
+
7
+ def initialize(file)
8
+ raise "Expecting a file object as an argument" unless file.is_a?(File)
9
+
10
+ # Ensure there are sufficient number of bytes to determine the
11
+ # signature.
12
+ if file.stat.size < minimum_bytes_for_determining_signature
13
+ puts "File too small to calculate signature"
14
+ return false
15
+ end
16
+
17
+ @file = file
18
+ process_file!
19
+ end
20
+
21
+ def starting_signature_bytes
22
+ 2
23
+ end
24
+
25
+ def valid?
26
+ @starting_signature == VALID_STARTING_SIGNATURE
27
+ end
28
+ private
29
+
30
+ def minimum_bytes_for_determining_signature
31
+ starting_signature_bytes
32
+ end
33
+
34
+ def process_file!
35
+ read_starting_signature!
36
+
37
+ # Ensure the file is closed after reading the starting signiture
38
+ # bytes
39
+ @file.close
40
+ end
41
+
42
+ def read_starting_signature!
43
+ @file.rewind
44
+ starting_bytes = @file.readpartial(starting_signature_bytes)
45
+ @starting_signature = starting_bytes.unpack("H*").first
46
+ end
47
+
48
+ end
49
+
50
+ #puts MagicGzipValidator.new(File.new('json.log', 'r')).valid?
51
+
52
+ #=> true
53
+
@@ -0,0 +1,59 @@
1
+ # not needed - Mutex is part of core lib:
2
+ #require 'thread'
3
+
4
+ class S3ClientFactory
5
+
6
+ def initialize(logger, options, aws_options_hash)
7
+ @logger = logger
8
+ @aws_options_hash = aws_options_hash
9
+ @s3_default_options = Hash[options[:s3_default_options].map { |k, v| [k.to_sym, v] }]
10
+ @aws_options_hash.merge!(@s3_default_options) unless @s3_default_options.empty?
11
+ @sts_client = Aws::STS::Client.new(region: options[:aws_region])
12
+ @credentials_by_bucket = options[:s3_credentials_by_bucket]
13
+ @region_by_bucket = options[:s3_region_by_bucket]
14
+ @logger.debug("Credentials by Bucket", :credentials => @credentials_by_bucket)
15
+ @default_session_name = options[:s3_role_session_name]
16
+ @clients_by_bucket = {}
17
+ @creation_mutex = Mutex.new
18
+ end
19
+
20
+ def get_s3_client(bucket_name)
21
+ bucket_symbol = bucket_name.to_sym
22
+ @creation_mutex.synchronize do
23
+ if @clients_by_bucket[bucket_symbol].nil?
24
+ options = @aws_options_hash.clone
25
+ unless @credentials_by_bucket[bucket_name].nil?
26
+ options.merge!(credentials: get_s3_auth(@credentials_by_bucket[bucket_name]))
27
+ end
28
+ unless @region_by_bucket[bucket_name].nil?
29
+ options.merge!(region: @region_by_bucket[bucket_name])
30
+ end
31
+ @clients_by_bucket[bucket_symbol] = Aws::S3::Client.new(options)
32
+ @logger.debug("Created a new S3 Client", :bucket_name => bucket_name, :client => @clients_by_bucket[bucket_symbol], :used_options => options)
33
+ end
34
+ end
35
+ # to be thread-safe, one uses this method like this:
36
+ # s3_client_factory.get_s3_client(my_s3_bucket) do
37
+ # ... do stuff ...
38
+ # end
39
+ yield @clients_by_bucket[bucket_symbol]
40
+ end
41
+
42
+ private
43
+
44
+ def get_s3_auth(credentials)
45
+ # reminder: these are auto-refreshing!
46
+ if credentials.key?('role')
47
+ @logger.debug("Assume Role", :role => credentials["role"])
48
+ return Aws::AssumeRoleCredentials.new(
49
+ client: @sts_client,
50
+ role_arn: credentials['role'],
51
+ role_session_name: @default_session_name
52
+ )
53
+ elsif credentials.key?('access_key_id') && credentials.key?('secret_access_key')
54
+ @logger.debug("Fetch credentials", :access_key => credentials['access_key_id'])
55
+ return Aws::Credentials.new(credentials['access_key_id'], credentials['secret_access_key'])
56
+ end
57
+ end
58
+
59
+ end # class
@@ -0,0 +1,57 @@
1
+ # encoding: utf-8
2
+ require 'fileutils'
3
+ require 'thread'
4
+
5
+ class S3Downloader
6
+
7
+ def initialize(logger, stop_semaphore, options)
8
+ @logger = logger
9
+ @stopped = stop_semaphore
10
+ @factory = options[:s3_client_factory]
11
+ @delete_on_success = options[:delete_on_success]
12
+ @include_object_properties = options[:include_object_properties]
13
+ end
14
+
15
+ def copy_s3object_to_disk(record)
16
+ # (from docs) WARNING:
17
+ # yielding data to a block disables retries of networking errors!
18
+ begin
19
+ @factory.get_s3_client(record[:bucket]) do |s3|
20
+ response = s3.get_object(
21
+ bucket: record[:bucket],
22
+ key: record[:key],
23
+ response_target: record[:local_file]
24
+ )
25
+ record[:s3_data] = response.to_h.keep_if { |key| @include_object_properties.include?(key) }
26
+ end
27
+ rescue Aws::S3::Errors::ServiceError => e
28
+ @logger.error("Unable to download file. Requeuing the message", :error => e, :record => record)
29
+ # prevent sqs message deletion
30
+ throw :skip_delete
31
+ end
32
+ throw :skip_delete if stop?
33
+ return true
34
+ end
35
+
36
+ def cleanup_local_object(record)
37
+ FileUtils.remove_entry_secure(record[:local_file], true) if ::File.exists?(record[:local_file])
38
+ rescue Exception => e
39
+ @logger.warn("Could not delete file", :file => record[:local_file], :error => e)
40
+ end
41
+
42
+ def cleanup_s3object(record)
43
+ return unless @delete_on_success
44
+ begin
45
+ @factory.get_s3_client(record[:bucket]) do |s3|
46
+ s3.delete_object(bucket: record[:bucket], key: record[:key])
47
+ end
48
+ rescue Exception => e
49
+ @logger.warn("Failed to delete s3 object", :record => record, :error => e)
50
+ end
51
+ end
52
+
53
+ def stop?
54
+ @stopped.value
55
+ end
56
+
57
+ end # class
@@ -0,0 +1,143 @@
1
+ # LogProcessor:
2
+ # reads and decodes locally available file with log lines
3
+ # and creates LogStash events from these
4
+ require 'logstash/inputs/mime/magic_gzip_validator'
5
+ require 'pathname'
6
+
7
+ module LogProcessor
8
+
9
+ def self.included(base)
10
+ base.extend(self)
11
+ end
12
+
13
+ def process(record, logstash_event_queue)
14
+ file = record[:local_file]
15
+ codec = @codec_factory.get_codec(record)
16
+ folder = record[:folder]
17
+ type = @type_by_folder.fetch(record[:bucket],{})[folder]
18
+ metadata = {}
19
+ line_count = 0
20
+ event_count = 0
21
+ #start_time = Time.now
22
+ file_t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
23
+ read_file(file) do |line|
24
+ line_count += 1
25
+ if stop?
26
+ @logger.warn("[#{Thread.current[:name]}] Abort reading in the middle of the file, we will read it again when logstash is started")
27
+ throw :skip_delete
28
+ end
29
+ begin
30
+ codec.decode(line) do |event|
31
+ event_count += 1
32
+ decorate_event(event, metadata, type, record[:key], record[:bucket], record[:s3_data])
33
+ #event_time = Time.now #PROFILING
34
+ #event.set("[@metadata][progress][begin]", start_time)
35
+ #event.set("[@metadata][progress][index_time]", event_time)
36
+ #event.set("[@metadata][progress][line]", line_count)
37
+ logstash_event_queue << event
38
+ end
39
+ rescue Exception => e
40
+ @logger.error("[#{Thread.current[:name]}] Unable to decode line", :line => line, :error => e)
41
+ end
42
+ end
43
+ file_t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
44
+ processing_time = (file_t1 - file_t0)
45
+ #@logger.warn("[#{Thread.current[:name]}] Completed long running File ( took #{processing_time} ) s", file: record[:key], events: event_count, processing_time: processing_time ) if processing_time > 600.0 #PROFILING
46
+ # ensure any stateful codecs (such as multi-line ) are flushed to the queue
47
+ codec.flush do |event|
48
+ event_count += 1
49
+ decorate_event(event, metadata, type, record[:key], record[:bucket], record[:s3_data])
50
+ @logger.debug("[#{Thread.current[:name]}] Flushing an incomplete event", :event => event.to_s)
51
+ logstash_event_queue << event
52
+ end
53
+ # signal completion:
54
+ return true
55
+ end
56
+
57
+ private
58
+
59
+ def decorate_event(event, metadata, type, key, bucket, s3_data)
60
+ if event_is_metadata?(event)
61
+ @logger.debug('Updating the current cloudfront metadata', :event => event)
62
+ update_metadata(metadata, event)
63
+ else
64
+ # type by folder - set before "decorate()" enforces default
65
+ event.set('type', type) if type and ! event.include?('type')
66
+ decorate(event)
67
+
68
+ event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
69
+ event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
70
+
71
+ event.set("[@metadata][s3]", s3_data)
72
+ event.set("[@metadata][s3][object_key]", key)
73
+ event.set("[@metadata][s3][bucket_name]", bucket)
74
+ event.set("[@metadata][s3][object_folder]", get_object_folder(key))
75
+
76
+ end
77
+ end
78
+
79
+ def gzip?(filename)
80
+ return true if filename.end_with?('.gz','.gzip')
81
+ MagicGzipValidator.new(File.new(filename, 'rb')).valid?
82
+ rescue Exception => e
83
+ @logger.warn("Problem while gzip detection", :error => e)
84
+ end
85
+
86
+ def read_file(filename)
87
+ zipped = gzip?(filename)
88
+ completed = false
89
+ file_stream = FileInputStream.new(filename)
90
+ if zipped
91
+ gzip_stream = GZIPInputStream.new(file_stream)
92
+ decoder = InputStreamReader.new(gzip_stream, 'UTF-8')
93
+ else
94
+ decoder = InputStreamReader.new(file_stream, 'UTF-8')
95
+ end
96
+ buffered = BufferedReader.new(decoder)
97
+
98
+ while (data = buffered.readLine())
99
+ line = StringBuilder.new(data).append("\n")
100
+ yield(line.toString())
101
+ end
102
+ completed = true
103
+ rescue ZipException => e
104
+ @logger.error("Gzip codec: We cannot uncompress the gzip file", :filename => filename, :error => e)
105
+ ensure
106
+ buffered.close unless buffered.nil?
107
+ decoder.close unless decoder.nil?
108
+ gzip_stream.close unless gzip_stream.nil?
109
+ file_stream.close unless file_stream.nil?
110
+
111
+ unless completed
112
+ @logger.warn("[#{Thread.current[:name]}] Incomplete message in read_file. We´ll throw skip_delete.", :filename => filename)
113
+ throw :skip_delete
114
+ end
115
+
116
+ end
117
+
118
+ def event_is_metadata?(event)
119
+ return false unless event.get("message").class == String
120
+ line = event.get("message")
121
+ version_metadata?(line) || fields_metadata?(line)
122
+ end
123
+
124
+ def version_metadata?(line)
125
+ line.start_with?('#Version: ')
126
+ end
127
+
128
+ def fields_metadata?(line)
129
+ line.start_with?('#Fields: ')
130
+ end
131
+
132
+ def update_metadata(metadata, event)
133
+ line = event.get('message').strip
134
+
135
+ if version_metadata?(line)
136
+ metadata[:cloudfront_version] = line.split(/#Version: (.+)/).last
137
+ end
138
+
139
+ if fields_metadata?(line)
140
+ metadata[:cloudfront_fields] = line.split(/#Fields: (.+)/).last
141
+ end
142
+ end
143
+ end
@@ -0,0 +1,22 @@
1
+ # This patch was stolen from logstash-plugins/logstash-output-s3#102.
2
+ #
3
+ # This patch is a workaround for a JRuby issue which has been fixed in JRuby
4
+ # 9000, but not in JRuby 1.7. See https://github.com/jruby/jruby/issues/3645
5
+ # and https://github.com/jruby/jruby/issues/3920. This is necessary because the
6
+ # `aws-sdk` is doing tricky name discovery to generate the correct error class.
7
+ #
8
+ # As per https://github.com/aws/aws-sdk-ruby/issues/1301#issuecomment-261115960,
9
+ # this patch may be short-lived anyway.
10
+ require 'aws-sdk'
11
+
12
+ begin
13
+ old_stderr = $stderr
14
+ $stderr = StringIO.new
15
+
16
+ module Aws
17
+ const_set(:S3, Aws::S3)
18
+ const_set(:SQS, Aws::SQS)
19
+ end
20
+ ensure
21
+ $stderr = old_stderr
22
+ end
@@ -0,0 +1,218 @@
1
+ # MessagePoller:
2
+ # polling loop fetches messages from source queue and invokes
3
+ # the provided code block on them
4
+ require 'json'
5
+ require 'cgi'
6
+
7
+ class SqsPoller
8
+
9
+ # queue poller options we want to set explicitly
10
+ DEFAULT_OPTIONS = {
11
+ # we query one message at a time, so we can ensure correct error
12
+ # handling if we can't download a single file correctly
13
+ # (we will throw :skip_delete if download size isn't correct to allow
14
+ # for processing the event again later, so make sure to set a reasonable
15
+ # "DefaultVisibilityTimeout" for your queue so that there's enough time
16
+ # to process the log files!)
17
+ max_number_of_messages: 1,
18
+ visibility_timeout: 10,
19
+ # long polling; by default we use the queue's setting.
20
+ # A good value is 10 seconds to to balance between a quick logstash
21
+ # shutdown and fewer api calls.
22
+ wait_time_seconds: nil,
23
+ #attribute_names: ["All"], # Receive all available built-in message attributes.
24
+ #message_attribute_names: ["All"], # Receive any custom message attributes.
25
+ skip_delete: false,
26
+ }
27
+
28
+ # only needed in "run_with_backoff":
29
+ BACKOFF_SLEEP_TIME = 1
30
+ BACKOFF_FACTOR = 2
31
+ MAX_TIME_BEFORE_GIVING_UP = 60
32
+ # only needed in "preprocess":
33
+ EVENT_SOURCE = 'aws:s3'
34
+ EVENT_TYPE = 'ObjectCreated'
35
+
36
+ # initialization and setup happens once, outside the threads:
37
+ #
38
+ def initialize(logger, stop_semaphore, poller_options = {}, client_options = {}, aws_options_hash)
39
+ @logger = logger
40
+ @stopped = stop_semaphore
41
+ @queue = client_options[:sqs_queue]
42
+ @from_sns = client_options[:from_sns]
43
+ @max_processing_time = client_options[:max_processing_time]
44
+ @sqs_delete_on_failure = client_options[:sqs_delete_on_failure]
45
+ @options = DEFAULT_OPTIONS.merge(poller_options)
46
+ begin
47
+ @logger.info("Registering SQS input", :queue => @queue)
48
+ sqs_client = Aws::SQS::Client.new(aws_options_hash)
49
+ if uri?(@queue)
50
+ queue_url = @queue
51
+ else
52
+ queue_url = sqs_client.get_queue_url({
53
+ queue_name: @queue,
54
+ queue_owner_aws_account_id: client_options[:queue_owner_aws_account_id]
55
+ }).queue_url
56
+ end
57
+
58
+ @poller = Aws::SQS::QueuePoller.new(queue_url,
59
+ :client => sqs_client
60
+ )
61
+ @logger.info("[#{Thread.current[:name]}] connected to queue.", :queue_url => queue_url)
62
+ rescue Aws::SQS::Errors::ServiceError => e
63
+ @logger.error("Cannot establish connection to Amazon SQS", :error => e)
64
+ raise LogStash::ConfigurationError, "Verify the SQS queue name and your credentials"
65
+ end
66
+ end
67
+
68
+ # this is called by every worker thread:
69
+ def run() # not (&block) - pass explicitly (use yield below)
70
+ # per-thread timer to extend visibility if necessary
71
+ extender = nil
72
+ message_backoff = (@options[:visibility_timeout] * 95).to_f / 100.0
73
+ new_visibility = 2 * @options[:visibility_timeout]
74
+
75
+ # "shutdown handler":
76
+ @poller.before_request do |_|
77
+ if stop?
78
+ # kill visibility extender thread if active?
79
+ extender.kill if extender
80
+ extender = nil
81
+ @logger.warn('issuing :stop_polling on "stop?" signal', :queue => @queue)
82
+ # this can take up to "Receive Message Wait Time" (of the sqs queue) seconds to be recognized
83
+ throw :stop_polling
84
+ end
85
+ end
86
+
87
+ run_with_backoff do
88
+ message_count = 0 #PROFILING
89
+ @poller.poll(@options) do |message|
90
+ message_count += 1 #PROFILING
91
+ message_t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
92
+ # auto-increase the timeout if processing takes too long:
93
+ poller_thread = Thread.current
94
+ extender = Thread.new do
95
+ while new_visibility < @max_processing_time do
96
+
97
+ sleep message_backoff
98
+ begin
99
+ @poller.change_message_visibility_timeout(message, new_visibility)
100
+ @logger.warn("[#{Thread.current[:name]}] Extended visibility for a long running message", :visibility => new_visibility) if new_visibility > 600.0
101
+ new_visibility += message_backoff
102
+ rescue Aws::SQS::Errors::InvalidParameterValue => e
103
+ @logger.debug("Extending visibility failed for message", :error => e)
104
+ else
105
+ @logger.debug("[#{Thread.current[:name]}] Extended visibility for message", :visibility => new_visibility) #PROFILING
106
+ end
107
+ end
108
+ @logger.error("[#{Thread.current[:name]}] Maximum visibility reached! We will delete this message from queue!")
109
+ @poller.delete_message(message) if @sqs_delete_on_failure
110
+ poller_thread.kill
111
+ end
112
+ extender[:name] = "#{Thread.current[:name]}/extender" #PROFILING
113
+ failed = false
114
+ record_count = 0
115
+ begin
116
+ message_completed = catch(:skip_delete) do
117
+ preprocess(message) do |record|
118
+ record_count += 1
119
+ extender[:name] = "#{Thread.current[:name]}/extender/#{record[:key]}" #PROFILING
120
+ yield(record)
121
+ end
122
+ end
123
+ rescue Exception => e
124
+ @logger.warn("Error in poller loop", :error => e)
125
+ @logger.warn("Backtrace:\n\t#{e.backtrace.join("\n\t")}")
126
+ failed = true
127
+ end
128
+ message_t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
129
+ unless message_completed
130
+ @logger.debug("[#{Thread.current[:name]}] uncompleted message at the end of poller loop. We´ll throw skip_delete.", :message_count => message_count)
131
+ extender.run if extender
132
+ end
133
+ # at this time the extender has either fired or is obsolete
134
+ extender.kill if extender
135
+ extender = nil
136
+ throw :skip_delete if failed or ! message_completed
137
+ #@logger.info("[#{Thread.current[:name]}] completed message.", :message => message_count)
138
+ end
139
+ end
140
+ end
141
+
142
+ private
143
+
144
+ def stop?
145
+ @stopped.value
146
+ end
147
+
148
+ # Customization to parse CrowdStrike Falcon Data Replicator queue messages
149
+ def preprocess(message)
150
+ @logger.debug("Inside FDR Queue Preprocess: Start", :event => message)
151
+ payload = JSON.parse(message.body)
152
+ payload = JSON.parse(payload['Message']) if @from_sns
153
+ @logger.debug("Payload in Preprocess: ", :payload => payload)
154
+
155
+ # skip files other than aidmaster data (for now)
156
+ return nil unless payload['files'] and (payload['pathPrefix'].start_with?("data/") or payload['pathPrefix'].start_with?("aidmaster/"))
157
+
158
+ bucket = payload['bucket']
159
+ payload['files'].each do |file|
160
+ @logger.debug("We found a file", :file => file)
161
+ # in case there are any events with Records that aren't s3 object-created events and can't therefore be
162
+ # processed by this plugin, we will skip them and remove them from queue
163
+ # if record['eventSource'] == EVENT_SOURCE and record['eventName'].start_with?(EVENT_TYPE) then
164
+ key = file['path']
165
+ @logger.debug("We found a file", :key => key)
166
+ size = file['size']
167
+ @logger.debug("We found a file", :size => size)
168
+
169
+ yield({
170
+ bucket: bucket,
171
+ key: key,
172
+ size: size,
173
+ folder: get_object_path(key)
174
+ })
175
+ #end
176
+ end
177
+ end
178
+
179
+
180
+
181
+ # Runs an AWS request inside a Ruby block with an exponential backoff in case
182
+ # we experience a ServiceError.
183
+ # @param [Integer] max_time maximum amount of time to sleep before giving up.
184
+ # @param [Integer] sleep_time the initial amount of time to sleep before retrying.
185
+ # instead of requiring
186
+ # @param [Block] block Ruby code block to execute
187
+ # and then doing a "block.call",
188
+ # we yield to the passed block.
189
+ def run_with_backoff(max_time = MAX_TIME_BEFORE_GIVING_UP, sleep_time = BACKOFF_SLEEP_TIME)
190
+ next_sleep = sleep_time
191
+ begin
192
+ yield
193
+ next_sleep = sleep_time
194
+ rescue Aws::SQS::Errors::ServiceError => e
195
+ @logger.warn("Aws::SQS::Errors::ServiceError ... retrying SQS request with exponential backoff", :queue => @queue, :sleep_time => sleep_time, :error => e)
196
+ sleep(next_sleep)
197
+ next_sleep = next_sleep > max_time ? sleep_time : sleep_time * BACKOFF_FACTOR
198
+ retry
199
+ end
200
+ end
201
+
202
+ def uri?(string)
203
+ uri = URI.parse(string)
204
+ %w( http https ).include?(uri.scheme)
205
+ rescue URI::BadURIError
206
+ false
207
+ rescue URI::InvalidURIError
208
+ false
209
+ end
210
+
211
+
212
+ def get_object_path(key)
213
+ folder = ::File.dirname(key)
214
+ return '' if folder == '.'
215
+ return folder
216
+ end
217
+
218
+ end