logstash-input-crowdstrike_fdr 2.1.2

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,53 @@
1
+ class MagicGzipValidator
2
+ attr_reader :file
3
+ attr_reader :starting_signature
4
+
5
+ VALID_STARTING_SIGNATURE = "1f8b"
6
+
7
+ def initialize(file)
8
+ raise "Expecting a file object as an argument" unless file.is_a?(File)
9
+
10
+ # Ensure there are sufficient number of bytes to determine the
11
+ # signature.
12
+ if file.stat.size < minimum_bytes_for_determining_signature
13
+ puts "File too small to calculate signature"
14
+ return false
15
+ end
16
+
17
+ @file = file
18
+ process_file!
19
+ end
20
+
21
+ def starting_signature_bytes
22
+ 2
23
+ end
24
+
25
+ def valid?
26
+ @starting_signature == VALID_STARTING_SIGNATURE
27
+ end
28
+ private
29
+
30
+ def minimum_bytes_for_determining_signature
31
+ starting_signature_bytes
32
+ end
33
+
34
+ def process_file!
35
+ read_starting_signature!
36
+
37
+ # Ensure the file is closed after reading the starting signiture
38
+ # bytes
39
+ @file.close
40
+ end
41
+
42
+ def read_starting_signature!
43
+ @file.rewind
44
+ starting_bytes = @file.readpartial(starting_signature_bytes)
45
+ @starting_signature = starting_bytes.unpack("H*").first
46
+ end
47
+
48
+ end
49
+
50
+ #puts MagicGzipValidator.new(File.new('json.log', 'r')).valid?
51
+
52
+ #=> true
53
+
@@ -0,0 +1,59 @@
1
+ # not needed - Mutex is part of core lib:
2
+ #require 'thread'
3
+
4
+ class S3ClientFactory
5
+
6
+ def initialize(logger, options, aws_options_hash)
7
+ @logger = logger
8
+ @aws_options_hash = aws_options_hash
9
+ @s3_default_options = Hash[options[:s3_default_options].map { |k, v| [k.to_sym, v] }]
10
+ @aws_options_hash.merge!(@s3_default_options) unless @s3_default_options.empty?
11
+ @sts_client = Aws::STS::Client.new(region: options[:aws_region])
12
+ @credentials_by_bucket = options[:s3_credentials_by_bucket]
13
+ @region_by_bucket = options[:s3_region_by_bucket]
14
+ @logger.debug("Credentials by Bucket", :credentials => @credentials_by_bucket)
15
+ @default_session_name = options[:s3_role_session_name]
16
+ @clients_by_bucket = {}
17
+ @creation_mutex = Mutex.new
18
+ end
19
+
20
+ def get_s3_client(bucket_name)
21
+ bucket_symbol = bucket_name.to_sym
22
+ @creation_mutex.synchronize do
23
+ if @clients_by_bucket[bucket_symbol].nil?
24
+ options = @aws_options_hash.clone
25
+ unless @credentials_by_bucket[bucket_name].nil?
26
+ options.merge!(credentials: get_s3_auth(@credentials_by_bucket[bucket_name]))
27
+ end
28
+ unless @region_by_bucket[bucket_name].nil?
29
+ options.merge!(region: @region_by_bucket[bucket_name])
30
+ end
31
+ @clients_by_bucket[bucket_symbol] = Aws::S3::Client.new(options)
32
+ @logger.debug("Created a new S3 Client", :bucket_name => bucket_name, :client => @clients_by_bucket[bucket_symbol], :used_options => options)
33
+ end
34
+ end
35
+ # to be thread-safe, one uses this method like this:
36
+ # s3_client_factory.get_s3_client(my_s3_bucket) do
37
+ # ... do stuff ...
38
+ # end
39
+ yield @clients_by_bucket[bucket_symbol]
40
+ end
41
+
42
+ private
43
+
44
+ def get_s3_auth(credentials)
45
+ # reminder: these are auto-refreshing!
46
+ if credentials.key?('role')
47
+ @logger.debug("Assume Role", :role => credentials["role"])
48
+ return Aws::AssumeRoleCredentials.new(
49
+ client: @sts_client,
50
+ role_arn: credentials['role'],
51
+ role_session_name: @default_session_name
52
+ )
53
+ elsif credentials.key?('access_key_id') && credentials.key?('secret_access_key')
54
+ @logger.debug("Fetch credentials", :access_key => credentials['access_key_id'])
55
+ return Aws::Credentials.new(credentials['access_key_id'], credentials['secret_access_key'])
56
+ end
57
+ end
58
+
59
+ end # class
@@ -0,0 +1,57 @@
1
+ # encoding: utf-8
2
+ require 'fileutils'
3
+ require 'thread'
4
+
5
+ class S3Downloader
6
+
7
+ def initialize(logger, stop_semaphore, options)
8
+ @logger = logger
9
+ @stopped = stop_semaphore
10
+ @factory = options[:s3_client_factory]
11
+ @delete_on_success = options[:delete_on_success]
12
+ @include_object_properties = options[:include_object_properties]
13
+ end
14
+
15
+ def copy_s3object_to_disk(record)
16
+ # (from docs) WARNING:
17
+ # yielding data to a block disables retries of networking errors!
18
+ begin
19
+ @factory.get_s3_client(record[:bucket]) do |s3|
20
+ response = s3.get_object(
21
+ bucket: record[:bucket],
22
+ key: record[:key],
23
+ response_target: record[:local_file]
24
+ )
25
+ record[:s3_data] = response.to_h.keep_if { |key| @include_object_properties.include?(key) }
26
+ end
27
+ rescue Aws::S3::Errors::ServiceError => e
28
+ @logger.error("Unable to download file. Requeuing the message", :error => e, :record => record)
29
+ # prevent sqs message deletion
30
+ throw :skip_delete
31
+ end
32
+ throw :skip_delete if stop?
33
+ return true
34
+ end
35
+
36
+ def cleanup_local_object(record)
37
+ FileUtils.remove_entry_secure(record[:local_file], true) if ::File.exists?(record[:local_file])
38
+ rescue Exception => e
39
+ @logger.warn("Could not delete file", :file => record[:local_file], :error => e)
40
+ end
41
+
42
+ def cleanup_s3object(record)
43
+ return unless @delete_on_success
44
+ begin
45
+ @factory.get_s3_client(record[:bucket]) do |s3|
46
+ s3.delete_object(bucket: record[:bucket], key: record[:key])
47
+ end
48
+ rescue Exception => e
49
+ @logger.warn("Failed to delete s3 object", :record => record, :error => e)
50
+ end
51
+ end
52
+
53
+ def stop?
54
+ @stopped.value
55
+ end
56
+
57
+ end # class
@@ -0,0 +1,143 @@
1
+ # LogProcessor:
2
+ # reads and decodes locally available file with log lines
3
+ # and creates LogStash events from these
4
+ require 'logstash/inputs/mime/magic_gzip_validator'
5
+ require 'pathname'
6
+
7
+ module LogProcessor
8
+
9
+ def self.included(base)
10
+ base.extend(self)
11
+ end
12
+
13
+ def process(record, logstash_event_queue)
14
+ file = record[:local_file]
15
+ codec = @codec_factory.get_codec(record)
16
+ folder = record[:folder]
17
+ type = @type_by_folder.fetch(record[:bucket],{})[folder]
18
+ metadata = {}
19
+ line_count = 0
20
+ event_count = 0
21
+ #start_time = Time.now
22
+ file_t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
23
+ read_file(file) do |line|
24
+ line_count += 1
25
+ if stop?
26
+ @logger.warn("[#{Thread.current[:name]}] Abort reading in the middle of the file, we will read it again when logstash is started")
27
+ throw :skip_delete
28
+ end
29
+ begin
30
+ codec.decode(line) do |event|
31
+ event_count += 1
32
+ decorate_event(event, metadata, type, record[:key], record[:bucket], record[:s3_data])
33
+ #event_time = Time.now #PROFILING
34
+ #event.set("[@metadata][progress][begin]", start_time)
35
+ #event.set("[@metadata][progress][index_time]", event_time)
36
+ #event.set("[@metadata][progress][line]", line_count)
37
+ logstash_event_queue << event
38
+ end
39
+ rescue Exception => e
40
+ @logger.error("[#{Thread.current[:name]}] Unable to decode line", :line => line, :error => e)
41
+ end
42
+ end
43
+ file_t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
44
+ processing_time = (file_t1 - file_t0)
45
+ #@logger.warn("[#{Thread.current[:name]}] Completed long running File ( took #{processing_time} ) s", file: record[:key], events: event_count, processing_time: processing_time ) if processing_time > 600.0 #PROFILING
46
+ # ensure any stateful codecs (such as multi-line ) are flushed to the queue
47
+ codec.flush do |event|
48
+ event_count += 1
49
+ decorate_event(event, metadata, type, record[:key], record[:bucket], record[:s3_data])
50
+ @logger.debug("[#{Thread.current[:name]}] Flushing an incomplete event", :event => event.to_s)
51
+ logstash_event_queue << event
52
+ end
53
+ # signal completion:
54
+ return true
55
+ end
56
+
57
+ private
58
+
59
+ def decorate_event(event, metadata, type, key, bucket, s3_data)
60
+ if event_is_metadata?(event)
61
+ @logger.debug('Updating the current cloudfront metadata', :event => event)
62
+ update_metadata(metadata, event)
63
+ else
64
+ # type by folder - set before "decorate()" enforces default
65
+ event.set('type', type) if type and ! event.include?('type')
66
+ decorate(event)
67
+
68
+ event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
69
+ event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
70
+
71
+ event.set("[@metadata][s3]", s3_data)
72
+ event.set("[@metadata][s3][object_key]", key)
73
+ event.set("[@metadata][s3][bucket_name]", bucket)
74
+ event.set("[@metadata][s3][object_folder]", get_object_folder(key))
75
+
76
+ end
77
+ end
78
+
79
+ def gzip?(filename)
80
+ return true if filename.end_with?('.gz','.gzip')
81
+ MagicGzipValidator.new(File.new(filename, 'rb')).valid?
82
+ rescue Exception => e
83
+ @logger.warn("Problem while gzip detection", :error => e)
84
+ end
85
+
86
+ def read_file(filename)
87
+ zipped = gzip?(filename)
88
+ completed = false
89
+ file_stream = FileInputStream.new(filename)
90
+ if zipped
91
+ gzip_stream = GZIPInputStream.new(file_stream)
92
+ decoder = InputStreamReader.new(gzip_stream, 'UTF-8')
93
+ else
94
+ decoder = InputStreamReader.new(file_stream, 'UTF-8')
95
+ end
96
+ buffered = BufferedReader.new(decoder)
97
+
98
+ while (data = buffered.readLine())
99
+ line = StringBuilder.new(data).append("\n")
100
+ yield(line.toString())
101
+ end
102
+ completed = true
103
+ rescue ZipException => e
104
+ @logger.error("Gzip codec: We cannot uncompress the gzip file", :filename => filename, :error => e)
105
+ ensure
106
+ buffered.close unless buffered.nil?
107
+ decoder.close unless decoder.nil?
108
+ gzip_stream.close unless gzip_stream.nil?
109
+ file_stream.close unless file_stream.nil?
110
+
111
+ unless completed
112
+ @logger.warn("[#{Thread.current[:name]}] Incomplete message in read_file. We´ll throw skip_delete.", :filename => filename)
113
+ throw :skip_delete
114
+ end
115
+
116
+ end
117
+
118
+ def event_is_metadata?(event)
119
+ return false unless event.get("message").class == String
120
+ line = event.get("message")
121
+ version_metadata?(line) || fields_metadata?(line)
122
+ end
123
+
124
+ def version_metadata?(line)
125
+ line.start_with?('#Version: ')
126
+ end
127
+
128
+ def fields_metadata?(line)
129
+ line.start_with?('#Fields: ')
130
+ end
131
+
132
+ def update_metadata(metadata, event)
133
+ line = event.get('message').strip
134
+
135
+ if version_metadata?(line)
136
+ metadata[:cloudfront_version] = line.split(/#Version: (.+)/).last
137
+ end
138
+
139
+ if fields_metadata?(line)
140
+ metadata[:cloudfront_fields] = line.split(/#Fields: (.+)/).last
141
+ end
142
+ end
143
+ end
@@ -0,0 +1,22 @@
1
+ # This patch was stolen from logstash-plugins/logstash-output-s3#102.
2
+ #
3
+ # This patch is a workaround for a JRuby issue which has been fixed in JRuby
4
+ # 9000, but not in JRuby 1.7. See https://github.com/jruby/jruby/issues/3645
5
+ # and https://github.com/jruby/jruby/issues/3920. This is necessary because the
6
+ # `aws-sdk` is doing tricky name discovery to generate the correct error class.
7
+ #
8
+ # As per https://github.com/aws/aws-sdk-ruby/issues/1301#issuecomment-261115960,
9
+ # this patch may be short-lived anyway.
10
+ require 'aws-sdk'
11
+
12
+ begin
13
+ old_stderr = $stderr
14
+ $stderr = StringIO.new
15
+
16
+ module Aws
17
+ const_set(:S3, Aws::S3)
18
+ const_set(:SQS, Aws::SQS)
19
+ end
20
+ ensure
21
+ $stderr = old_stderr
22
+ end
@@ -0,0 +1,218 @@
1
+ # MessagePoller:
2
+ # polling loop fetches messages from source queue and invokes
3
+ # the provided code block on them
4
+ require 'json'
5
+ require 'cgi'
6
+
7
+ class SqsPoller
8
+
9
+ # queue poller options we want to set explicitly
10
+ DEFAULT_OPTIONS = {
11
+ # we query one message at a time, so we can ensure correct error
12
+ # handling if we can't download a single file correctly
13
+ # (we will throw :skip_delete if download size isn't correct to allow
14
+ # for processing the event again later, so make sure to set a reasonable
15
+ # "DefaultVisibilityTimeout" for your queue so that there's enough time
16
+ # to process the log files!)
17
+ max_number_of_messages: 1,
18
+ visibility_timeout: 10,
19
+ # long polling; by default we use the queue's setting.
20
+ # A good value is 10 seconds to to balance between a quick logstash
21
+ # shutdown and fewer api calls.
22
+ wait_time_seconds: nil,
23
+ #attribute_names: ["All"], # Receive all available built-in message attributes.
24
+ #message_attribute_names: ["All"], # Receive any custom message attributes.
25
+ skip_delete: false,
26
+ }
27
+
28
+ # only needed in "run_with_backoff":
29
+ BACKOFF_SLEEP_TIME = 1
30
+ BACKOFF_FACTOR = 2
31
+ MAX_TIME_BEFORE_GIVING_UP = 60
32
+ # only needed in "preprocess":
33
+ EVENT_SOURCE = 'aws:s3'
34
+ EVENT_TYPE = 'ObjectCreated'
35
+
36
+ # initialization and setup happens once, outside the threads:
37
+ #
38
+ def initialize(logger, stop_semaphore, poller_options = {}, client_options = {}, aws_options_hash)
39
+ @logger = logger
40
+ @stopped = stop_semaphore
41
+ @queue = client_options[:sqs_queue]
42
+ @from_sns = client_options[:from_sns]
43
+ @max_processing_time = client_options[:max_processing_time]
44
+ @sqs_delete_on_failure = client_options[:sqs_delete_on_failure]
45
+ @options = DEFAULT_OPTIONS.merge(poller_options)
46
+ begin
47
+ @logger.info("Registering SQS input", :queue => @queue)
48
+ sqs_client = Aws::SQS::Client.new(aws_options_hash)
49
+ if uri?(@queue)
50
+ queue_url = @queue
51
+ else
52
+ queue_url = sqs_client.get_queue_url({
53
+ queue_name: @queue,
54
+ queue_owner_aws_account_id: client_options[:queue_owner_aws_account_id]
55
+ }).queue_url
56
+ end
57
+
58
+ @poller = Aws::SQS::QueuePoller.new(queue_url,
59
+ :client => sqs_client
60
+ )
61
+ @logger.info("[#{Thread.current[:name]}] connected to queue.", :queue_url => queue_url)
62
+ rescue Aws::SQS::Errors::ServiceError => e
63
+ @logger.error("Cannot establish connection to Amazon SQS", :error => e)
64
+ raise LogStash::ConfigurationError, "Verify the SQS queue name and your credentials"
65
+ end
66
+ end
67
+
68
+ # this is called by every worker thread:
69
+ def run() # not (&block) - pass explicitly (use yield below)
70
+ # per-thread timer to extend visibility if necessary
71
+ extender = nil
72
+ message_backoff = (@options[:visibility_timeout] * 95).to_f / 100.0
73
+ new_visibility = 2 * @options[:visibility_timeout]
74
+
75
+ # "shutdown handler":
76
+ @poller.before_request do |_|
77
+ if stop?
78
+ # kill visibility extender thread if active?
79
+ extender.kill if extender
80
+ extender = nil
81
+ @logger.warn('issuing :stop_polling on "stop?" signal', :queue => @queue)
82
+ # this can take up to "Receive Message Wait Time" (of the sqs queue) seconds to be recognized
83
+ throw :stop_polling
84
+ end
85
+ end
86
+
87
+ run_with_backoff do
88
+ message_count = 0 #PROFILING
89
+ @poller.poll(@options) do |message|
90
+ message_count += 1 #PROFILING
91
+ message_t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
92
+ # auto-increase the timeout if processing takes too long:
93
+ poller_thread = Thread.current
94
+ extender = Thread.new do
95
+ while new_visibility < @max_processing_time do
96
+
97
+ sleep message_backoff
98
+ begin
99
+ @poller.change_message_visibility_timeout(message, new_visibility)
100
+ @logger.warn("[#{Thread.current[:name]}] Extended visibility for a long running message", :visibility => new_visibility) if new_visibility > 600.0
101
+ new_visibility += message_backoff
102
+ rescue Aws::SQS::Errors::InvalidParameterValue => e
103
+ @logger.debug("Extending visibility failed for message", :error => e)
104
+ else
105
+ @logger.debug("[#{Thread.current[:name]}] Extended visibility for message", :visibility => new_visibility) #PROFILING
106
+ end
107
+ end
108
+ @logger.error("[#{Thread.current[:name]}] Maximum visibility reached! We will delete this message from queue!")
109
+ @poller.delete_message(message) if @sqs_delete_on_failure
110
+ poller_thread.kill
111
+ end
112
+ extender[:name] = "#{Thread.current[:name]}/extender" #PROFILING
113
+ failed = false
114
+ record_count = 0
115
+ begin
116
+ message_completed = catch(:skip_delete) do
117
+ preprocess(message) do |record|
118
+ record_count += 1
119
+ extender[:name] = "#{Thread.current[:name]}/extender/#{record[:key]}" #PROFILING
120
+ yield(record)
121
+ end
122
+ end
123
+ rescue Exception => e
124
+ @logger.warn("Error in poller loop", :error => e)
125
+ @logger.warn("Backtrace:\n\t#{e.backtrace.join("\n\t")}")
126
+ failed = true
127
+ end
128
+ message_t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
129
+ unless message_completed
130
+ @logger.debug("[#{Thread.current[:name]}] uncompleted message at the end of poller loop. We´ll throw skip_delete.", :message_count => message_count)
131
+ extender.run if extender
132
+ end
133
+ # at this time the extender has either fired or is obsolete
134
+ extender.kill if extender
135
+ extender = nil
136
+ throw :skip_delete if failed or ! message_completed
137
+ #@logger.info("[#{Thread.current[:name]}] completed message.", :message => message_count)
138
+ end
139
+ end
140
+ end
141
+
142
+ private
143
+
144
+ def stop?
145
+ @stopped.value
146
+ end
147
+
148
+ # Customization to parse CrowdStrike Falcon Data Replicator queue messages
149
+ def preprocess(message)
150
+ @logger.debug("Inside FDR Queue Preprocess: Start", :event => message)
151
+ payload = JSON.parse(message.body)
152
+ payload = JSON.parse(payload['Message']) if @from_sns
153
+ @logger.debug("Payload in Preprocess: ", :payload => payload)
154
+
155
+ # skip files other than aidmaster data (for now)
156
+ return nil unless payload['files'] and (payload['pathPrefix'].start_with?("data/") or payload['pathPrefix'].start_with?("aidmaster/"))
157
+
158
+ bucket = payload['bucket']
159
+ payload['files'].each do |file|
160
+ @logger.debug("We found a file", :file => file)
161
+ # in case there are any events with Records that aren't s3 object-created events and can't therefore be
162
+ # processed by this plugin, we will skip them and remove them from queue
163
+ # if record['eventSource'] == EVENT_SOURCE and record['eventName'].start_with?(EVENT_TYPE) then
164
+ key = file['path']
165
+ @logger.debug("We found a file", :key => key)
166
+ size = file['size']
167
+ @logger.debug("We found a file", :size => size)
168
+
169
+ yield({
170
+ bucket: bucket,
171
+ key: key,
172
+ size: size,
173
+ folder: get_object_path(key)
174
+ })
175
+ #end
176
+ end
177
+ end
178
+
179
+
180
+
181
+ # Runs an AWS request inside a Ruby block with an exponential backoff in case
182
+ # we experience a ServiceError.
183
+ # @param [Integer] max_time maximum amount of time to sleep before giving up.
184
+ # @param [Integer] sleep_time the initial amount of time to sleep before retrying.
185
+ # instead of requiring
186
+ # @param [Block] block Ruby code block to execute
187
+ # and then doing a "block.call",
188
+ # we yield to the passed block.
189
+ def run_with_backoff(max_time = MAX_TIME_BEFORE_GIVING_UP, sleep_time = BACKOFF_SLEEP_TIME)
190
+ next_sleep = sleep_time
191
+ begin
192
+ yield
193
+ next_sleep = sleep_time
194
+ rescue Aws::SQS::Errors::ServiceError => e
195
+ @logger.warn("Aws::SQS::Errors::ServiceError ... retrying SQS request with exponential backoff", :queue => @queue, :sleep_time => sleep_time, :error => e)
196
+ sleep(next_sleep)
197
+ next_sleep = next_sleep > max_time ? sleep_time : sleep_time * BACKOFF_FACTOR
198
+ retry
199
+ end
200
+ end
201
+
202
+ def uri?(string)
203
+ uri = URI.parse(string)
204
+ %w( http https ).include?(uri.scheme)
205
+ rescue URI::BadURIError
206
+ false
207
+ rescue URI::InvalidURIError
208
+ false
209
+ end
210
+
211
+
212
+ def get_object_path(key)
213
+ folder = ::File.dirname(key)
214
+ return '' if folder == '.'
215
+ return folder
216
+ end
217
+
218
+ end