logstash-output-awslogs 1.1.2 → 1.1.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (5) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +4 -0
  3. data/lib/logstash/outputs/awslogs.rb +93 -52
  4. data/logstash-output-awslogs.gemspec +2 -2
  5. metadata +29 -23
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3f29bdc8b680f033bac338ae4613bc72e73403fbc7664c4c8eba78a0f497fdf3
4
- data.tar.gz: 9b1e38924b1a1994ef71b69a4475f01d99e72345182b5aa1e8a0194b84cf98d6
3
+ metadata.gz: 5d89bf2683352b8d8e1dfb360760e5129db4d0bcfb447bea11101c3dc7648b11
4
+ data.tar.gz: 64e3749306d9da99ff81ca006bfb2e7c34255981b75119ee238d00b149e379f9
5
5
  SHA512:
6
- metadata.gz: 9900328521269d53376e645216989c7c01647f26c1290b8dc88edd5732521a0555fadeb00ad0f4b9674c8b5336b6b550bf62715e92e3be5cd4ec3b5142292cca
7
- data.tar.gz: 51d83fb4c44a8850b848dbc4900f96db80c0fa2521b7b9cf4726bfed3cf97ef7cb72d45ea69413d8f54062fc81773789d58c07177349690fa3766f9b92a7c548
6
+ metadata.gz: fc23ff3640ec9526027a3a1de59329dc00fedf2722602eca9e71ac4081007ad029e38391fe40a9895821aecd3e0f938c7f4f92c9f3139bee3a6410a640e8fffc
7
+ data.tar.gz: ce9e61a5e8d17bdb8aaa3a61bce0d7510f7547bc01327579263a9494cf110a8646d271d11b087bf915f035aa0de107f1a35eaa798f36306a546b9648ced32427
data/CHANGELOG.md CHANGED
@@ -3,3 +3,7 @@
3
3
 
4
4
  ## 1.1.1
5
5
  - First fully working version.
6
+
7
+ ## 1.1.3
8
+ - Fixed working with big batches
9
+ - Sequence token removed
data/lib/logstash/outputs/awslogs.rb CHANGED
@@ -4,7 +4,7 @@ require 'logstash/outputs/base'
4
4
  require 'logstash/namespace'
5
5
  require 'logstash/plugin_mixins/aws_config'
6
6
  require 'aws-sdk-cloudwatchlogs'
7
- require 'json'
7
+ require 'time'
8
8
 
9
9
 
10
10
  class LogStash::Outputs::Awslogs < LogStash::Outputs::Base
@@ -12,75 +12,116 @@ class LogStash::Outputs::Awslogs < LogStash::Outputs::Base
12
12
 
13
13
  config_name 'awslogs'
14
14
  default :codec, 'line'
15
+ concurrency :shared
15
16
 
17
+ PER_EVENT_OVERHEAD = 26
18
+ MAX_BATCH_SIZE = 1024 * 1024
19
+ MIN_DELAY = 0.2
20
+
21
+ # Log group to send event to
16
22
  config :log_group_name, validate: :string, required: true
23
+ # Logs stream to send event to
17
24
  config :log_stream_name, validate: :string, required: true
25
+ # Message to be sent. Fields interpolation supported. By default the whole event will be sent as a json object
26
+ config :message, validate: :string, default: ""
18
27
 
19
28
  public
20
29
  def register
21
30
  @client = Aws::CloudWatchLogs::Client.new(aws_options_hash)
22
- @next_sequence_tokens = {}
31
+ @last_flush = Time.now.to_f
23
32
  end # def register
24
33
 
25
34
  public
26
35
  def multi_receive(events)
27
- to_send = {}
36
+ send_batches = form_event_batches(events)
37
+ send_batches.each do |batch|
38
+ put_events(batch)
39
+ end
40
+ end
41
+
42
+ private
43
+ def put_events(batch)
44
+ begin
45
+ delay = MIN_DELAY - (Time.now.to_f - @last_flush)
46
+ sleep(delay) if delay > 0
47
+ @client.put_log_events(
48
+ {
49
+ log_group_name: batch[:log_group],
50
+ log_stream_name: batch[:log_stream],
51
+ log_events: batch[:log_events]
52
+ }
53
+ )
54
+ rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
55
+ @logger.info('AWSLogs: Will create log group/stream and retry')
56
+ begin
57
+ @client.create_log_group({log_group_name: batch[:log_group]})
58
+ rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
59
+ @logger.info("AWSLogs: Log group #{batch[:log_group]} already exists")
60
+ end
61
+ begin
62
+ @client.create_log_stream({log_group_name: batch[:log_group], log_stream_name: batch[:log_stream]})
63
+ rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
64
+ @logger.info("AWSLogs: Log stream #{batch[:log_stream]} already exists")
65
+ end
66
+ retry
67
+ end
68
+ rescue Aws::CloudWatchLogs::Errors::LimitExceededException
69
+ @logger.info("AWSLogs: Rate limit exceeded, retrying")
70
+ retry
71
+ end
72
+ @last_flush = Time.now.to_f
73
+ end
28
74
 
29
- events.each do |event|
75
+ private
76
+ def form_event_batches(events_arr)
77
+ batches = []
78
+ events_by_stream_and_group = {}
79
+ log_events = events_arr.sort_by { |event| event.timestamp.time.to_f}
80
+ log_events.each do |event|
30
81
  event_log_stream_name = event.sprintf(log_stream_name)
31
82
  event_log_group_name = event.sprintf(log_group_name)
32
83
 
33
- next_sequence_token_key = [event_log_group_name, event_log_stream_name]
34
- unless to_send.keys.include? next_sequence_token_key
35
- to_send.store(next_sequence_token_key, [])
84
+ sort_key = [event_log_group_name, event_log_stream_name]
85
+ unless events_by_stream_and_group.keys.include? sort_key
86
+ events_by_stream_and_group.store(sort_key, [])
87
+ end
88
+ if message.empty?
89
+ events_by_stream_and_group[sort_key].push(
90
+ timestamp: (event.timestamp.time.to_f * 1000).to_int,
91
+ message: event.to_hash.sort.to_h.to_json
92
+ )
93
+ else
94
+ events_by_stream_and_group[sort_key].push(
95
+ timestamp: (event.timestamp.time.to_f * 1000).to_int,
96
+ message: event.sprintf(message)
97
+ )
36
98
  end
37
- to_send[next_sequence_token_key].push(
38
- timestamp: (event.timestamp.time.to_f * 1000).to_int,
39
- message: event.to_hash.sort.to_h.to_json
40
- )
41
99
  end
42
-
43
-
44
- to_send.each do |event_log_names, log_events|
45
- event_log_group_name = event_log_names[0]
46
- event_log_stream_name = event_log_names[1]
47
- next_sequence_token_key = [event_log_group_name, event_log_stream_name]
48
-
49
- log_events.sort_by!{ |event| event[:timestamp] }
50
-
51
- ident_opts = {
52
- log_group_name: event_log_group_name,
53
- log_stream_name: event_log_stream_name
100
+ events_by_stream_and_group.each do |key, value|
101
+ temp_batch = {
102
+ log_group: key[0],
103
+ log_stream: key[1],
104
+ size: 0,
105
+ log_events: []
54
106
  }
55
- send_opts = ident_opts.merge(
56
- log_events: log_events
57
- )
58
- if @next_sequence_tokens.keys.include? next_sequence_token_key
59
- send_opts[:sequence_token] = @next_sequence_tokens[next_sequence_token_key]
60
- end
61
- begin
62
- resp = @client.put_log_events(send_opts)
63
- @next_sequence_tokens.store(next_sequence_token_key, resp.next_sequence_token)
64
- rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
65
- @logger.info('Will create log group/stream and retry')
66
- begin
67
- @client.create_log_group({log_group_name: send_opts[:log_group_name]})
68
- rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
69
- @logger.info("Log group #{send_opts[:log_group_name]} already exists")
70
- end
71
- begin
72
- @client.create_log_stream({log_group_name: send_opts[:log_group_name], log_stream_name: send_opts[:log_stream_name]})
73
- rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
74
- @logger.info("Log stream #{send_opts[:log_stream_name]} already exists")
107
+ value.each do |log_event|
108
+ if log_event[:message].bytesize + PER_EVENT_OVERHEAD + temp_batch[:size] < MAX_BATCH_SIZE
109
+ temp_batch[:size] += log_event[:message].bytesize + PER_EVENT_OVERHEAD
110
+ temp_batch[:log_events] << log_event
111
+ else
112
+ batches << temp_batch
113
+ temp_batch = {
114
+ log_group: key[0],
115
+ log_stream: key[1],
116
+ size: 0,
117
+ log_events: []
118
+ }
119
+ temp_batch[:size] += log_event[:message].bytesize + PER_EVENT_OVERHEAD
120
+ temp_batch[:log_events] << log_event
75
121
  end
76
- retry
77
- rescue Aws::CloudWatchLogs::Errors::InvalidSequenceTokenException => e
78
- send_opts[:sequence_token] = e.expected_sequence_token
79
- retry
80
- rescue Aws::CloudWatchLogs::Errors::ThrottlingException
81
- @logger.info('Logs throttling, retry')
82
- retry
83
122
  end
123
+ batches << temp_batch
84
124
  end
85
- end # def multi_receive_encodeds
86
- end # class LogStash::Outputs::Awslogs
125
+ batches
126
+ end
127
+ end
data/logstash-output-awslogs.gemspec CHANGED
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-awslogs'
3
- s.version = '1.1.2'
3
+ s.version = '1.1.4'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = 'Writes events to AWS CloudWatch logs.'
6
6
  s.homepage = 'https://github.com/Anarhyst266/logstash-output-awslogs'
@@ -17,7 +17,7 @@ Gem::Specification.new do |s|
17
17
  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
18
18
 
19
19
  # Gem dependencies
20
- s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
20
+ s.add_runtime_dependency "logstash-core-plugin-api", ">= 2.1.12", "<= 2.99"
21
21
  s.add_runtime_dependency "logstash-codec-plain"
22
22
  s.add_runtime_dependency "aws-sdk-cloudwatchlogs", '~> 1'
23
23
  s.add_runtime_dependency "logstash-integration-aws"
metadata CHANGED
@@ -1,100 +1,106 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-awslogs
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.2
4
+ version: 1.1.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Anton Klyba
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-12-12 00:00:00.000000000 Z
11
+ date: 2024-01-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: logstash-core-plugin-api
15
14
  requirement: !ruby/object:Gem::Requirement
16
15
  requirements:
17
- - - "~>"
16
+ - - ">="
18
17
  - !ruby/object:Gem::Version
19
- version: '2.0'
20
- type: :runtime
18
+ version: 2.1.12
19
+ - - "<="
20
+ - !ruby/object:Gem::Version
21
+ version: '2.99'
22
+ name: logstash-core-plugin-api
21
23
  prerelease: false
24
+ type: :runtime
22
25
  version_requirements: !ruby/object:Gem::Requirement
23
26
  requirements:
24
- - - "~>"
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: 2.1.12
30
+ - - "<="
25
31
  - !ruby/object:Gem::Version
26
- version: '2.0'
32
+ version: '2.99'
27
33
  - !ruby/object:Gem::Dependency
28
- name: logstash-codec-plain
29
34
  requirement: !ruby/object:Gem::Requirement
30
35
  requirements:
31
36
  - - ">="
32
37
  - !ruby/object:Gem::Version
33
38
  version: '0'
34
- type: :runtime
39
+ name: logstash-codec-plain
35
40
  prerelease: false
41
+ type: :runtime
36
42
  version_requirements: !ruby/object:Gem::Requirement
37
43
  requirements:
38
44
  - - ">="
39
45
  - !ruby/object:Gem::Version
40
46
  version: '0'
41
47
  - !ruby/object:Gem::Dependency
42
- name: aws-sdk-cloudwatchlogs
43
48
  requirement: !ruby/object:Gem::Requirement
44
49
  requirements:
45
50
  - - "~>"
46
51
  - !ruby/object:Gem::Version
47
52
  version: '1'
48
- type: :runtime
53
+ name: aws-sdk-cloudwatchlogs
49
54
  prerelease: false
55
+ type: :runtime
50
56
  version_requirements: !ruby/object:Gem::Requirement
51
57
  requirements:
52
58
  - - "~>"
53
59
  - !ruby/object:Gem::Version
54
60
  version: '1'
55
61
  - !ruby/object:Gem::Dependency
56
- name: logstash-integration-aws
57
62
  requirement: !ruby/object:Gem::Requirement
58
63
  requirements:
59
64
  - - ">="
60
65
  - !ruby/object:Gem::Version
61
66
  version: '0'
62
- type: :runtime
67
+ name: logstash-integration-aws
63
68
  prerelease: false
69
+ type: :runtime
64
70
  version_requirements: !ruby/object:Gem::Requirement
65
71
  requirements:
66
72
  - - ">="
67
73
  - !ruby/object:Gem::Version
68
74
  version: '0'
69
75
  - !ruby/object:Gem::Dependency
70
- name: json
71
76
  requirement: !ruby/object:Gem::Requirement
72
77
  requirements:
73
78
  - - ">="
74
79
  - !ruby/object:Gem::Version
75
80
  version: '0'
76
- type: :runtime
81
+ name: json
77
82
  prerelease: false
83
+ type: :runtime
78
84
  version_requirements: !ruby/object:Gem::Requirement
79
85
  requirements:
80
86
  - - ">="
81
87
  - !ruby/object:Gem::Version
82
88
  version: '0'
83
89
  - !ruby/object:Gem::Dependency
84
- name: logstash-devutils
85
90
  requirement: !ruby/object:Gem::Requirement
86
91
  requirements:
87
92
  - - ">="
88
93
  - !ruby/object:Gem::Version
89
94
  version: '0'
90
- type: :development
95
+ name: logstash-devutils
91
96
  prerelease: false
97
+ type: :development
92
98
  version_requirements: !ruby/object:Gem::Requirement
93
99
  requirements:
94
100
  - - ">="
95
101
  - !ruby/object:Gem::Version
96
102
  version: '0'
97
- description:
103
+ description:
98
104
  email: anarhyst266+gems@gmail.com
99
105
  executables: []
100
106
  extensions: []
@@ -114,7 +120,7 @@ licenses:
114
120
  metadata:
115
121
  logstash_plugin: 'true'
116
122
  logstash_group: output
117
- post_install_message:
123
+ post_install_message:
118
124
  rdoc_options: []
119
125
  require_paths:
120
126
  - lib
@@ -129,8 +135,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
129
135
  - !ruby/object:Gem::Version
130
136
  version: '0'
131
137
  requirements: []
132
- rubygems_version: 3.0.3.1
133
- signing_key:
138
+ rubygems_version: 3.3.26
139
+ signing_key:
134
140
  specification_version: 4
135
141
  summary: Writes events to AWS CloudWatch logs.
136
142
  test_files: []