logstash-input-crowdstrike_fdr 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +141 -0
- data/CONTRIBUTORS +14 -0
- data/Gemfile +11 -0
- data/LICENSE +13 -0
- data/NOTICE.TXT +5 -0
- data/README.md +147 -0
- data/lib/logstash/inputs/codec_factory.rb +37 -0
- data/lib/logstash/inputs/crowdstrike_fdr.rb +343 -0
- data/lib/logstash/inputs/mime/magic_gzip_validator.rb +53 -0
- data/lib/logstash/inputs/s3/client_factory.rb +59 -0
- data/lib/logstash/inputs/s3/downloader.rb +57 -0
- data/lib/logstash/inputs/s3snssqs/log_processor.rb +143 -0
- data/lib/logstash/inputs/s3sqs/patch.rb +22 -0
- data/lib/logstash/inputs/sqs/poller.rb +218 -0
- data/logstash-input-crowdstrike_fdr.gemspec +28 -0
- data/spec/inputs/crowdstrike_fdr_spec.rb +66 -0
- metadata +141 -0
@@ -0,0 +1,53 @@
|
|
1
|
+
class MagicGzipValidator
|
2
|
+
attr_reader :file
|
3
|
+
attr_reader :starting_signature
|
4
|
+
|
5
|
+
VALID_STARTING_SIGNATURE = "1f8b"
|
6
|
+
|
7
|
+
def initialize(file)
|
8
|
+
raise "Expecting a file object as an argument" unless file.is_a?(File)
|
9
|
+
|
10
|
+
# Ensure there are sufficient number of bytes to determine the
|
11
|
+
# signature.
|
12
|
+
if file.stat.size < minimum_bytes_for_determining_signature
|
13
|
+
puts "File too small to calculate signature"
|
14
|
+
return false
|
15
|
+
end
|
16
|
+
|
17
|
+
@file = file
|
18
|
+
process_file!
|
19
|
+
end
|
20
|
+
|
21
|
+
def starting_signature_bytes
|
22
|
+
2
|
23
|
+
end
|
24
|
+
|
25
|
+
def valid?
|
26
|
+
@starting_signature == VALID_STARTING_SIGNATURE
|
27
|
+
end
|
28
|
+
private
|
29
|
+
|
30
|
+
def minimum_bytes_for_determining_signature
|
31
|
+
starting_signature_bytes
|
32
|
+
end
|
33
|
+
|
34
|
+
def process_file!
|
35
|
+
read_starting_signature!
|
36
|
+
|
37
|
+
# Ensure the file is closed after reading the starting signiture
|
38
|
+
# bytes
|
39
|
+
@file.close
|
40
|
+
end
|
41
|
+
|
42
|
+
def read_starting_signature!
|
43
|
+
@file.rewind
|
44
|
+
starting_bytes = @file.readpartial(starting_signature_bytes)
|
45
|
+
@starting_signature = starting_bytes.unpack("H*").first
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
|
50
|
+
#puts MagicGzipValidator.new(File.new('json.log', 'r')).valid?
|
51
|
+
|
52
|
+
#=> true
|
53
|
+
|
@@ -0,0 +1,59 @@
|
|
1
|
+
# not needed - Mutex is part of core lib:
|
2
|
+
#require 'thread'
|
3
|
+
|
4
|
+
class S3ClientFactory
|
5
|
+
|
6
|
+
def initialize(logger, options, aws_options_hash)
|
7
|
+
@logger = logger
|
8
|
+
@aws_options_hash = aws_options_hash
|
9
|
+
@s3_default_options = Hash[options[:s3_default_options].map { |k, v| [k.to_sym, v] }]
|
10
|
+
@aws_options_hash.merge!(@s3_default_options) unless @s3_default_options.empty?
|
11
|
+
@sts_client = Aws::STS::Client.new(region: options[:aws_region])
|
12
|
+
@credentials_by_bucket = options[:s3_credentials_by_bucket]
|
13
|
+
@region_by_bucket = options[:s3_region_by_bucket]
|
14
|
+
@logger.debug("Credentials by Bucket", :credentials => @credentials_by_bucket)
|
15
|
+
@default_session_name = options[:s3_role_session_name]
|
16
|
+
@clients_by_bucket = {}
|
17
|
+
@creation_mutex = Mutex.new
|
18
|
+
end
|
19
|
+
|
20
|
+
def get_s3_client(bucket_name)
|
21
|
+
bucket_symbol = bucket_name.to_sym
|
22
|
+
@creation_mutex.synchronize do
|
23
|
+
if @clients_by_bucket[bucket_symbol].nil?
|
24
|
+
options = @aws_options_hash.clone
|
25
|
+
unless @credentials_by_bucket[bucket_name].nil?
|
26
|
+
options.merge!(credentials: get_s3_auth(@credentials_by_bucket[bucket_name]))
|
27
|
+
end
|
28
|
+
unless @region_by_bucket[bucket_name].nil?
|
29
|
+
options.merge!(region: @region_by_bucket[bucket_name])
|
30
|
+
end
|
31
|
+
@clients_by_bucket[bucket_symbol] = Aws::S3::Client.new(options)
|
32
|
+
@logger.debug("Created a new S3 Client", :bucket_name => bucket_name, :client => @clients_by_bucket[bucket_symbol], :used_options => options)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
# to be thread-safe, one uses this method like this:
|
36
|
+
# s3_client_factory.get_s3_client(my_s3_bucket) do
|
37
|
+
# ... do stuff ...
|
38
|
+
# end
|
39
|
+
yield @clients_by_bucket[bucket_symbol]
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
def get_s3_auth(credentials)
|
45
|
+
# reminder: these are auto-refreshing!
|
46
|
+
if credentials.key?('role')
|
47
|
+
@logger.debug("Assume Role", :role => credentials["role"])
|
48
|
+
return Aws::AssumeRoleCredentials.new(
|
49
|
+
client: @sts_client,
|
50
|
+
role_arn: credentials['role'],
|
51
|
+
role_session_name: @default_session_name
|
52
|
+
)
|
53
|
+
elsif credentials.key?('access_key_id') && credentials.key?('secret_access_key')
|
54
|
+
@logger.debug("Fetch credentials", :access_key => credentials['access_key_id'])
|
55
|
+
return Aws::Credentials.new(credentials['access_key_id'], credentials['secret_access_key'])
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
end # class
|
@@ -0,0 +1,57 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'fileutils'
|
3
|
+
require 'thread'
|
4
|
+
|
5
|
+
class S3Downloader
|
6
|
+
|
7
|
+
def initialize(logger, stop_semaphore, options)
|
8
|
+
@logger = logger
|
9
|
+
@stopped = stop_semaphore
|
10
|
+
@factory = options[:s3_client_factory]
|
11
|
+
@delete_on_success = options[:delete_on_success]
|
12
|
+
@include_object_properties = options[:include_object_properties]
|
13
|
+
end
|
14
|
+
|
15
|
+
def copy_s3object_to_disk(record)
|
16
|
+
# (from docs) WARNING:
|
17
|
+
# yielding data to a block disables retries of networking errors!
|
18
|
+
begin
|
19
|
+
@factory.get_s3_client(record[:bucket]) do |s3|
|
20
|
+
response = s3.get_object(
|
21
|
+
bucket: record[:bucket],
|
22
|
+
key: record[:key],
|
23
|
+
response_target: record[:local_file]
|
24
|
+
)
|
25
|
+
record[:s3_data] = response.to_h.keep_if { |key| @include_object_properties.include?(key) }
|
26
|
+
end
|
27
|
+
rescue Aws::S3::Errors::ServiceError => e
|
28
|
+
@logger.error("Unable to download file. Requeuing the message", :error => e, :record => record)
|
29
|
+
# prevent sqs message deletion
|
30
|
+
throw :skip_delete
|
31
|
+
end
|
32
|
+
throw :skip_delete if stop?
|
33
|
+
return true
|
34
|
+
end
|
35
|
+
|
36
|
+
def cleanup_local_object(record)
|
37
|
+
FileUtils.remove_entry_secure(record[:local_file], true) if ::File.exists?(record[:local_file])
|
38
|
+
rescue Exception => e
|
39
|
+
@logger.warn("Could not delete file", :file => record[:local_file], :error => e)
|
40
|
+
end
|
41
|
+
|
42
|
+
def cleanup_s3object(record)
|
43
|
+
return unless @delete_on_success
|
44
|
+
begin
|
45
|
+
@factory.get_s3_client(record[:bucket]) do |s3|
|
46
|
+
s3.delete_object(bucket: record[:bucket], key: record[:key])
|
47
|
+
end
|
48
|
+
rescue Exception => e
|
49
|
+
@logger.warn("Failed to delete s3 object", :record => record, :error => e)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def stop?
|
54
|
+
@stopped.value
|
55
|
+
end
|
56
|
+
|
57
|
+
end # class
|
@@ -0,0 +1,143 @@
|
|
1
|
+
# LogProcessor:
|
2
|
+
# reads and decodes locally available file with log lines
|
3
|
+
# and creates LogStash events from these
|
4
|
+
require 'logstash/inputs/mime/magic_gzip_validator'
|
5
|
+
require 'pathname'
|
6
|
+
|
7
|
+
module LogProcessor
|
8
|
+
|
9
|
+
def self.included(base)
|
10
|
+
base.extend(self)
|
11
|
+
end
|
12
|
+
|
13
|
+
def process(record, logstash_event_queue)
|
14
|
+
file = record[:local_file]
|
15
|
+
codec = @codec_factory.get_codec(record)
|
16
|
+
folder = record[:folder]
|
17
|
+
type = @type_by_folder.fetch(record[:bucket],{})[folder]
|
18
|
+
metadata = {}
|
19
|
+
line_count = 0
|
20
|
+
event_count = 0
|
21
|
+
#start_time = Time.now
|
22
|
+
file_t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
|
23
|
+
read_file(file) do |line|
|
24
|
+
line_count += 1
|
25
|
+
if stop?
|
26
|
+
@logger.warn("[#{Thread.current[:name]}] Abort reading in the middle of the file, we will read it again when logstash is started")
|
27
|
+
throw :skip_delete
|
28
|
+
end
|
29
|
+
begin
|
30
|
+
codec.decode(line) do |event|
|
31
|
+
event_count += 1
|
32
|
+
decorate_event(event, metadata, type, record[:key], record[:bucket], record[:s3_data])
|
33
|
+
#event_time = Time.now #PROFILING
|
34
|
+
#event.set("[@metadata][progress][begin]", start_time)
|
35
|
+
#event.set("[@metadata][progress][index_time]", event_time)
|
36
|
+
#event.set("[@metadata][progress][line]", line_count)
|
37
|
+
logstash_event_queue << event
|
38
|
+
end
|
39
|
+
rescue Exception => e
|
40
|
+
@logger.error("[#{Thread.current[:name]}] Unable to decode line", :line => line, :error => e)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
file_t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
|
44
|
+
processing_time = (file_t1 - file_t0)
|
45
|
+
#@logger.warn("[#{Thread.current[:name]}] Completed long running File ( took #{processing_time} ) s", file: record[:key], events: event_count, processing_time: processing_time ) if processing_time > 600.0 #PROFILING
|
46
|
+
# ensure any stateful codecs (such as multi-line ) are flushed to the queue
|
47
|
+
codec.flush do |event|
|
48
|
+
event_count += 1
|
49
|
+
decorate_event(event, metadata, type, record[:key], record[:bucket], record[:s3_data])
|
50
|
+
@logger.debug("[#{Thread.current[:name]}] Flushing an incomplete event", :event => event.to_s)
|
51
|
+
logstash_event_queue << event
|
52
|
+
end
|
53
|
+
# signal completion:
|
54
|
+
return true
|
55
|
+
end
|
56
|
+
|
57
|
+
private
|
58
|
+
|
59
|
+
def decorate_event(event, metadata, type, key, bucket, s3_data)
|
60
|
+
if event_is_metadata?(event)
|
61
|
+
@logger.debug('Updating the current cloudfront metadata', :event => event)
|
62
|
+
update_metadata(metadata, event)
|
63
|
+
else
|
64
|
+
# type by folder - set before "decorate()" enforces default
|
65
|
+
event.set('type', type) if type and ! event.include?('type')
|
66
|
+
decorate(event)
|
67
|
+
|
68
|
+
event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
|
69
|
+
event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
|
70
|
+
|
71
|
+
event.set("[@metadata][s3]", s3_data)
|
72
|
+
event.set("[@metadata][s3][object_key]", key)
|
73
|
+
event.set("[@metadata][s3][bucket_name]", bucket)
|
74
|
+
event.set("[@metadata][s3][object_folder]", get_object_folder(key))
|
75
|
+
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def gzip?(filename)
|
80
|
+
return true if filename.end_with?('.gz','.gzip')
|
81
|
+
MagicGzipValidator.new(File.new(filename, 'rb')).valid?
|
82
|
+
rescue Exception => e
|
83
|
+
@logger.warn("Problem while gzip detection", :error => e)
|
84
|
+
end
|
85
|
+
|
86
|
+
def read_file(filename)
|
87
|
+
zipped = gzip?(filename)
|
88
|
+
completed = false
|
89
|
+
file_stream = FileInputStream.new(filename)
|
90
|
+
if zipped
|
91
|
+
gzip_stream = GZIPInputStream.new(file_stream)
|
92
|
+
decoder = InputStreamReader.new(gzip_stream, 'UTF-8')
|
93
|
+
else
|
94
|
+
decoder = InputStreamReader.new(file_stream, 'UTF-8')
|
95
|
+
end
|
96
|
+
buffered = BufferedReader.new(decoder)
|
97
|
+
|
98
|
+
while (data = buffered.readLine())
|
99
|
+
line = StringBuilder.new(data).append("\n")
|
100
|
+
yield(line.toString())
|
101
|
+
end
|
102
|
+
completed = true
|
103
|
+
rescue ZipException => e
|
104
|
+
@logger.error("Gzip codec: We cannot uncompress the gzip file", :filename => filename, :error => e)
|
105
|
+
ensure
|
106
|
+
buffered.close unless buffered.nil?
|
107
|
+
decoder.close unless decoder.nil?
|
108
|
+
gzip_stream.close unless gzip_stream.nil?
|
109
|
+
file_stream.close unless file_stream.nil?
|
110
|
+
|
111
|
+
unless completed
|
112
|
+
@logger.warn("[#{Thread.current[:name]}] Incomplete message in read_file. We´ll throw skip_delete.", :filename => filename)
|
113
|
+
throw :skip_delete
|
114
|
+
end
|
115
|
+
|
116
|
+
end
|
117
|
+
|
118
|
+
def event_is_metadata?(event)
|
119
|
+
return false unless event.get("message").class == String
|
120
|
+
line = event.get("message")
|
121
|
+
version_metadata?(line) || fields_metadata?(line)
|
122
|
+
end
|
123
|
+
|
124
|
+
def version_metadata?(line)
|
125
|
+
line.start_with?('#Version: ')
|
126
|
+
end
|
127
|
+
|
128
|
+
def fields_metadata?(line)
|
129
|
+
line.start_with?('#Fields: ')
|
130
|
+
end
|
131
|
+
|
132
|
+
def update_metadata(metadata, event)
|
133
|
+
line = event.get('message').strip
|
134
|
+
|
135
|
+
if version_metadata?(line)
|
136
|
+
metadata[:cloudfront_version] = line.split(/#Version: (.+)/).last
|
137
|
+
end
|
138
|
+
|
139
|
+
if fields_metadata?(line)
|
140
|
+
metadata[:cloudfront_fields] = line.split(/#Fields: (.+)/).last
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# This patch was stolen from logstash-plugins/logstash-output-s3#102.
|
2
|
+
#
|
3
|
+
# This patch is a workaround for a JRuby issue which has been fixed in JRuby
|
4
|
+
# 9000, but not in JRuby 1.7. See https://github.com/jruby/jruby/issues/3645
|
5
|
+
# and https://github.com/jruby/jruby/issues/3920. This is necessary because the
|
6
|
+
# `aws-sdk` is doing tricky name discovery to generate the correct error class.
|
7
|
+
#
|
8
|
+
# As per https://github.com/aws/aws-sdk-ruby/issues/1301#issuecomment-261115960,
|
9
|
+
# this patch may be short-lived anyway.
|
10
|
+
require 'aws-sdk'
|
11
|
+
|
12
|
+
begin
|
13
|
+
old_stderr = $stderr
|
14
|
+
$stderr = StringIO.new
|
15
|
+
|
16
|
+
module Aws
|
17
|
+
const_set(:S3, Aws::S3)
|
18
|
+
const_set(:SQS, Aws::SQS)
|
19
|
+
end
|
20
|
+
ensure
|
21
|
+
$stderr = old_stderr
|
22
|
+
end
|
@@ -0,0 +1,218 @@
|
|
1
|
+
# MessagePoller:
|
2
|
+
# polling loop fetches messages from source queue and invokes
|
3
|
+
# the provided code block on them
|
4
|
+
require 'json'
|
5
|
+
require 'cgi'
|
6
|
+
|
7
|
+
class SqsPoller
|
8
|
+
|
9
|
+
# queue poller options we want to set explicitly
|
10
|
+
DEFAULT_OPTIONS = {
|
11
|
+
# we query one message at a time, so we can ensure correct error
|
12
|
+
# handling if we can't download a single file correctly
|
13
|
+
# (we will throw :skip_delete if download size isn't correct to allow
|
14
|
+
# for processing the event again later, so make sure to set a reasonable
|
15
|
+
# "DefaultVisibilityTimeout" for your queue so that there's enough time
|
16
|
+
# to process the log files!)
|
17
|
+
max_number_of_messages: 1,
|
18
|
+
visibility_timeout: 10,
|
19
|
+
# long polling; by default we use the queue's setting.
|
20
|
+
# A good value is 10 seconds to to balance between a quick logstash
|
21
|
+
# shutdown and fewer api calls.
|
22
|
+
wait_time_seconds: nil,
|
23
|
+
#attribute_names: ["All"], # Receive all available built-in message attributes.
|
24
|
+
#message_attribute_names: ["All"], # Receive any custom message attributes.
|
25
|
+
skip_delete: false,
|
26
|
+
}
|
27
|
+
|
28
|
+
# only needed in "run_with_backoff":
|
29
|
+
BACKOFF_SLEEP_TIME = 1
|
30
|
+
BACKOFF_FACTOR = 2
|
31
|
+
MAX_TIME_BEFORE_GIVING_UP = 60
|
32
|
+
# only needed in "preprocess":
|
33
|
+
EVENT_SOURCE = 'aws:s3'
|
34
|
+
EVENT_TYPE = 'ObjectCreated'
|
35
|
+
|
36
|
+
# initialization and setup happens once, outside the threads:
|
37
|
+
#
|
38
|
+
def initialize(logger, stop_semaphore, poller_options = {}, client_options = {}, aws_options_hash)
|
39
|
+
@logger = logger
|
40
|
+
@stopped = stop_semaphore
|
41
|
+
@queue = client_options[:sqs_queue]
|
42
|
+
@from_sns = client_options[:from_sns]
|
43
|
+
@max_processing_time = client_options[:max_processing_time]
|
44
|
+
@sqs_delete_on_failure = client_options[:sqs_delete_on_failure]
|
45
|
+
@options = DEFAULT_OPTIONS.merge(poller_options)
|
46
|
+
begin
|
47
|
+
@logger.info("Registering SQS input", :queue => @queue)
|
48
|
+
sqs_client = Aws::SQS::Client.new(aws_options_hash)
|
49
|
+
if uri?(@queue)
|
50
|
+
queue_url = @queue
|
51
|
+
else
|
52
|
+
queue_url = sqs_client.get_queue_url({
|
53
|
+
queue_name: @queue,
|
54
|
+
queue_owner_aws_account_id: client_options[:queue_owner_aws_account_id]
|
55
|
+
}).queue_url
|
56
|
+
end
|
57
|
+
|
58
|
+
@poller = Aws::SQS::QueuePoller.new(queue_url,
|
59
|
+
:client => sqs_client
|
60
|
+
)
|
61
|
+
@logger.info("[#{Thread.current[:name]}] connected to queue.", :queue_url => queue_url)
|
62
|
+
rescue Aws::SQS::Errors::ServiceError => e
|
63
|
+
@logger.error("Cannot establish connection to Amazon SQS", :error => e)
|
64
|
+
raise LogStash::ConfigurationError, "Verify the SQS queue name and your credentials"
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
# this is called by every worker thread:
|
69
|
+
def run() # not (&block) - pass explicitly (use yield below)
|
70
|
+
# per-thread timer to extend visibility if necessary
|
71
|
+
extender = nil
|
72
|
+
message_backoff = (@options[:visibility_timeout] * 95).to_f / 100.0
|
73
|
+
new_visibility = 2 * @options[:visibility_timeout]
|
74
|
+
|
75
|
+
# "shutdown handler":
|
76
|
+
@poller.before_request do |_|
|
77
|
+
if stop?
|
78
|
+
# kill visibility extender thread if active?
|
79
|
+
extender.kill if extender
|
80
|
+
extender = nil
|
81
|
+
@logger.warn('issuing :stop_polling on "stop?" signal', :queue => @queue)
|
82
|
+
# this can take up to "Receive Message Wait Time" (of the sqs queue) seconds to be recognized
|
83
|
+
throw :stop_polling
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
run_with_backoff do
|
88
|
+
message_count = 0 #PROFILING
|
89
|
+
@poller.poll(@options) do |message|
|
90
|
+
message_count += 1 #PROFILING
|
91
|
+
message_t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
|
92
|
+
# auto-increase the timeout if processing takes too long:
|
93
|
+
poller_thread = Thread.current
|
94
|
+
extender = Thread.new do
|
95
|
+
while new_visibility < @max_processing_time do
|
96
|
+
|
97
|
+
sleep message_backoff
|
98
|
+
begin
|
99
|
+
@poller.change_message_visibility_timeout(message, new_visibility)
|
100
|
+
@logger.warn("[#{Thread.current[:name]}] Extended visibility for a long running message", :visibility => new_visibility) if new_visibility > 600.0
|
101
|
+
new_visibility += message_backoff
|
102
|
+
rescue Aws::SQS::Errors::InvalidParameterValue => e
|
103
|
+
@logger.debug("Extending visibility failed for message", :error => e)
|
104
|
+
else
|
105
|
+
@logger.debug("[#{Thread.current[:name]}] Extended visibility for message", :visibility => new_visibility) #PROFILING
|
106
|
+
end
|
107
|
+
end
|
108
|
+
@logger.error("[#{Thread.current[:name]}] Maximum visibility reached! We will delete this message from queue!")
|
109
|
+
@poller.delete_message(message) if @sqs_delete_on_failure
|
110
|
+
poller_thread.kill
|
111
|
+
end
|
112
|
+
extender[:name] = "#{Thread.current[:name]}/extender" #PROFILING
|
113
|
+
failed = false
|
114
|
+
record_count = 0
|
115
|
+
begin
|
116
|
+
message_completed = catch(:skip_delete) do
|
117
|
+
preprocess(message) do |record|
|
118
|
+
record_count += 1
|
119
|
+
extender[:name] = "#{Thread.current[:name]}/extender/#{record[:key]}" #PROFILING
|
120
|
+
yield(record)
|
121
|
+
end
|
122
|
+
end
|
123
|
+
rescue Exception => e
|
124
|
+
@logger.warn("Error in poller loop", :error => e)
|
125
|
+
@logger.warn("Backtrace:\n\t#{e.backtrace.join("\n\t")}")
|
126
|
+
failed = true
|
127
|
+
end
|
128
|
+
message_t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
|
129
|
+
unless message_completed
|
130
|
+
@logger.debug("[#{Thread.current[:name]}] uncompleted message at the end of poller loop. We´ll throw skip_delete.", :message_count => message_count)
|
131
|
+
extender.run if extender
|
132
|
+
end
|
133
|
+
# at this time the extender has either fired or is obsolete
|
134
|
+
extender.kill if extender
|
135
|
+
extender = nil
|
136
|
+
throw :skip_delete if failed or ! message_completed
|
137
|
+
#@logger.info("[#{Thread.current[:name]}] completed message.", :message => message_count)
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
private
|
143
|
+
|
144
|
+
def stop?
|
145
|
+
@stopped.value
|
146
|
+
end
|
147
|
+
|
148
|
+
# Customization to parse CrowdStrike Falcon Data Replicator queue messages
|
149
|
+
def preprocess(message)
|
150
|
+
@logger.debug("Inside FDR Queue Preprocess: Start", :event => message)
|
151
|
+
payload = JSON.parse(message.body)
|
152
|
+
payload = JSON.parse(payload['Message']) if @from_sns
|
153
|
+
@logger.debug("Payload in Preprocess: ", :payload => payload)
|
154
|
+
|
155
|
+
# skip files other than aidmaster data (for now)
|
156
|
+
return nil unless payload['files'] and (payload['pathPrefix'].start_with?("data/") or payload['pathPrefix'].start_with?("aidmaster/"))
|
157
|
+
|
158
|
+
bucket = payload['bucket']
|
159
|
+
payload['files'].each do |file|
|
160
|
+
@logger.debug("We found a file", :file => file)
|
161
|
+
# in case there are any events with Records that aren't s3 object-created events and can't therefore be
|
162
|
+
# processed by this plugin, we will skip them and remove them from queue
|
163
|
+
# if record['eventSource'] == EVENT_SOURCE and record['eventName'].start_with?(EVENT_TYPE) then
|
164
|
+
key = file['path']
|
165
|
+
@logger.debug("We found a file", :key => key)
|
166
|
+
size = file['size']
|
167
|
+
@logger.debug("We found a file", :size => size)
|
168
|
+
|
169
|
+
yield({
|
170
|
+
bucket: bucket,
|
171
|
+
key: key,
|
172
|
+
size: size,
|
173
|
+
folder: get_object_path(key)
|
174
|
+
})
|
175
|
+
#end
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
|
180
|
+
|
181
|
+
# Runs an AWS request inside a Ruby block with an exponential backoff in case
|
182
|
+
# we experience a ServiceError.
|
183
|
+
# @param [Integer] max_time maximum amount of time to sleep before giving up.
|
184
|
+
# @param [Integer] sleep_time the initial amount of time to sleep before retrying.
|
185
|
+
# instead of requiring
|
186
|
+
# @param [Block] block Ruby code block to execute
|
187
|
+
# and then doing a "block.call",
|
188
|
+
# we yield to the passed block.
|
189
|
+
def run_with_backoff(max_time = MAX_TIME_BEFORE_GIVING_UP, sleep_time = BACKOFF_SLEEP_TIME)
|
190
|
+
next_sleep = sleep_time
|
191
|
+
begin
|
192
|
+
yield
|
193
|
+
next_sleep = sleep_time
|
194
|
+
rescue Aws::SQS::Errors::ServiceError => e
|
195
|
+
@logger.warn("Aws::SQS::Errors::ServiceError ... retrying SQS request with exponential backoff", :queue => @queue, :sleep_time => sleep_time, :error => e)
|
196
|
+
sleep(next_sleep)
|
197
|
+
next_sleep = next_sleep > max_time ? sleep_time : sleep_time * BACKOFF_FACTOR
|
198
|
+
retry
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
def uri?(string)
|
203
|
+
uri = URI.parse(string)
|
204
|
+
%w( http https ).include?(uri.scheme)
|
205
|
+
rescue URI::BadURIError
|
206
|
+
false
|
207
|
+
rescue URI::InvalidURIError
|
208
|
+
false
|
209
|
+
end
|
210
|
+
|
211
|
+
|
212
|
+
def get_object_path(key)
|
213
|
+
folder = ::File.dirname(key)
|
214
|
+
return '' if folder == '.'
|
215
|
+
return folder
|
216
|
+
end
|
217
|
+
|
218
|
+
end
|