logstash-output-awslogs 1.1.2 → 1.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +4 -0
  3. data/lib/logstash/outputs/awslogs.rb +93 -52
  4. data/logstash-output-awslogs.gemspec +2 -2
  5. metadata +29 -23
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3f29bdc8b680f033bac338ae4613bc72e73403fbc7664c4c8eba78a0f497fdf3
4
- data.tar.gz: 9b1e38924b1a1994ef71b69a4475f01d99e72345182b5aa1e8a0194b84cf98d6
3
+ metadata.gz: 5d89bf2683352b8d8e1dfb360760e5129db4d0bcfb447bea11101c3dc7648b11
4
+ data.tar.gz: 64e3749306d9da99ff81ca006bfb2e7c34255981b75119ee238d00b149e379f9
5
5
  SHA512:
6
- metadata.gz: 9900328521269d53376e645216989c7c01647f26c1290b8dc88edd5732521a0555fadeb00ad0f4b9674c8b5336b6b550bf62715e92e3be5cd4ec3b5142292cca
7
- data.tar.gz: 51d83fb4c44a8850b848dbc4900f96db80c0fa2521b7b9cf4726bfed3cf97ef7cb72d45ea69413d8f54062fc81773789d58c07177349690fa3766f9b92a7c548
6
+ metadata.gz: fc23ff3640ec9526027a3a1de59329dc00fedf2722602eca9e71ac4081007ad029e38391fe40a9895821aecd3e0f938c7f4f92c9f3139bee3a6410a640e8fffc
7
+ data.tar.gz: ce9e61a5e8d17bdb8aaa3a61bce0d7510f7547bc01327579263a9494cf110a8646d271d11b087bf915f035aa0de107f1a35eaa798f36306a546b9648ced32427
data/CHANGELOG.md CHANGED
@@ -3,3 +3,7 @@
3
3
 
4
4
  ## 1.1.1
5
5
  - First fully working version.
6
+
7
+ ## 1.1.3
8
+ - Fixed working with big batches
9
+ - Sequence token removed
data/lib/logstash/outputs/awslogs.rb CHANGED
@@ -4,7 +4,7 @@ require 'logstash/outputs/base'
4
4
  require 'logstash/namespace'
5
5
  require 'logstash/plugin_mixins/aws_config'
6
6
  require 'aws-sdk-cloudwatchlogs'
7
- require 'json'
7
+ require 'time'
8
8
 
9
9
 
10
10
  class LogStash::Outputs::Awslogs < LogStash::Outputs::Base
@@ -12,75 +12,116 @@ class LogStash::Outputs::Awslogs < LogStash::Outputs::Base
12
12
 
13
13
  config_name 'awslogs'
14
14
  default :codec, 'line'
15
+ concurrency :shared
15
16
 
17
+ PER_EVENT_OVERHEAD = 26
18
+ MAX_BATCH_SIZE = 1024 * 1024
19
+ MIN_DELAY = 0.2
20
+
21
+ # Log group to send event to
16
22
  config :log_group_name, validate: :string, required: true
23
+ # Logs stream to send event to
17
24
  config :log_stream_name, validate: :string, required: true
25
+ # Message to be sent. Fields interpolation supported. By default the whole event will be sent as a json object
26
+ config :message, validate: :string, default: ""
18
27
 
19
28
  public
20
29
  def register
21
30
  @client = Aws::CloudWatchLogs::Client.new(aws_options_hash)
22
- @next_sequence_tokens = {}
31
+ @last_flush = Time.now.to_f
23
32
  end # def register
24
33
 
25
34
  public
26
35
  def multi_receive(events)
27
- to_send = {}
36
+ send_batches = form_event_batches(events)
37
+ send_batches.each do |batch|
38
+ put_events(batch)
39
+ end
40
+ end
41
+
42
+ private
43
+ def put_events(batch)
44
+ begin
45
+ delay = MIN_DELAY - (Time.now.to_f - @last_flush)
46
+ sleep(delay) if delay > 0
47
+ @client.put_log_events(
48
+ {
49
+ log_group_name: batch[:log_group],
50
+ log_stream_name: batch[:log_stream],
51
+ log_events: batch[:log_events]
52
+ }
53
+ )
54
+ rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
55
+ @logger.info('AWSLogs: Will create log group/stream and retry')
56
+ begin
57
+ @client.create_log_group({log_group_name: batch[:log_group]})
58
+ rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
59
+ @logger.info("AWSLogs: Log group #{batch[:log_group]} already exists")
60
+ end
61
+ begin
62
+ @client.create_log_stream({log_group_name: batch[:log_group], log_stream_name: batch[:log_stream]})
63
+ rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
64
+ @logger.info("AWSLogs: Log stream #{batch[:log_stream]} already exists")
65
+ end
66
+ retry
67
+ end
68
+ rescue Aws::CloudWatchLogs::Errors::LimitExceededException
69
+ @logger.info("AWSLogs: Rate limit exceeded, retrying")
70
+ retry
71
+ end
72
+ @last_flush = Time.now.to_f
73
+ end
28
74
 
29
- events.each do |event|
75
+ private
76
+ def form_event_batches(events_arr)
77
+ batches = []
78
+ events_by_stream_and_group = {}
79
+ log_events = events_arr.sort_by { |event| event.timestamp.time.to_f}
80
+ log_events.each do |event|
30
81
  event_log_stream_name = event.sprintf(log_stream_name)
31
82
  event_log_group_name = event.sprintf(log_group_name)
32
83
 
33
- next_sequence_token_key = [event_log_group_name, event_log_stream_name]
34
- unless to_send.keys.include? next_sequence_token_key
35
- to_send.store(next_sequence_token_key, [])
84
+ sort_key = [event_log_group_name, event_log_stream_name]
85
+ unless events_by_stream_and_group.keys.include? sort_key
86
+ events_by_stream_and_group.store(sort_key, [])
87
+ end
88
+ if message.empty?
89
+ events_by_stream_and_group[sort_key].push(
90
+ timestamp: (event.timestamp.time.to_f * 1000).to_int,
91
+ message: event.to_hash.sort.to_h.to_json
92
+ )
93
+ else
94
+ events_by_stream_and_group[sort_key].push(
95
+ timestamp: (event.timestamp.time.to_f * 1000).to_int,
96
+ message: event.sprintf(message)
97
+ )
36
98
  end
37
- to_send[next_sequence_token_key].push(
38
- timestamp: (event.timestamp.time.to_f * 1000).to_int,
39
- message: event.to_hash.sort.to_h.to_json
40
- )
41
99
  end
42
-
43
-
44
- to_send.each do |event_log_names, log_events|
45
- event_log_group_name = event_log_names[0]
46
- event_log_stream_name = event_log_names[1]
47
- next_sequence_token_key = [event_log_group_name, event_log_stream_name]
48
-
49
- log_events.sort_by!{ |event| event[:timestamp] }
50
-
51
- ident_opts = {
52
- log_group_name: event_log_group_name,
53
- log_stream_name: event_log_stream_name
100
+ events_by_stream_and_group.each do |key, value|
101
+ temp_batch = {
102
+ log_group: key[0],
103
+ log_stream: key[1],
104
+ size: 0,
105
+ log_events: []
54
106
  }
55
- send_opts = ident_opts.merge(
56
- log_events: log_events
57
- )
58
- if @next_sequence_tokens.keys.include? next_sequence_token_key
59
- send_opts[:sequence_token] = @next_sequence_tokens[next_sequence_token_key]
60
- end
61
- begin
62
- resp = @client.put_log_events(send_opts)
63
- @next_sequence_tokens.store(next_sequence_token_key, resp.next_sequence_token)
64
- rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
65
- @logger.info('Will create log group/stream and retry')
66
- begin
67
- @client.create_log_group({log_group_name: send_opts[:log_group_name]})
68
- rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
69
- @logger.info("Log group #{send_opts[:log_group_name]} already exists")
70
- end
71
- begin
72
- @client.create_log_stream({log_group_name: send_opts[:log_group_name], log_stream_name: send_opts[:log_stream_name]})
73
- rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
74
- @logger.info("Log stream #{send_opts[:log_stream_name]} already exists")
107
+ value.each do |log_event|
108
+ if log_event[:message].bytesize + PER_EVENT_OVERHEAD + temp_batch[:size] < MAX_BATCH_SIZE
109
+ temp_batch[:size] += log_event[:message].bytesize + PER_EVENT_OVERHEAD
110
+ temp_batch[:log_events] << log_event
111
+ else
112
+ batches << temp_batch
113
+ temp_batch = {
114
+ log_group: key[0],
115
+ log_stream: key[1],
116
+ size: 0,
117
+ log_events: []
118
+ }
119
+ temp_batch[:size] += log_event[:message].bytesize + PER_EVENT_OVERHEAD
120
+ temp_batch[:log_events] << log_event
75
121
  end
76
- retry
77
- rescue Aws::CloudWatchLogs::Errors::InvalidSequenceTokenException => e
78
- send_opts[:sequence_token] = e.expected_sequence_token
79
- retry
80
- rescue Aws::CloudWatchLogs::Errors::ThrottlingException
81
- @logger.info('Logs throttling, retry')
82
- retry
83
122
  end
123
+ batches << temp_batch
84
124
  end
85
- end # def multi_receive_encodeds
86
- end # class LogStash::Outputs::Awslogs
125
+ batches
126
+ end
127
+ end
data/logstash-output-awslogs.gemspec CHANGED
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-awslogs'
3
- s.version = '1.1.2'
3
+ s.version = '1.1.4'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = 'Writes events to AWS CloudWatch logs.'
6
6
  s.homepage = 'https://github.com/Anarhyst266/logstash-output-awslogs'
@@ -17,7 +17,7 @@ Gem::Specification.new do |s|
17
17
  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
18
18
 
19
19
  # Gem dependencies
20
- s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
20
+ s.add_runtime_dependency "logstash-core-plugin-api", ">= 2.1.12", "<= 2.99"
21
21
  s.add_runtime_dependency "logstash-codec-plain"
22
22
  s.add_runtime_dependency "aws-sdk-cloudwatchlogs", '~> 1'
23
23
  s.add_runtime_dependency "logstash-integration-aws"
metadata CHANGED
@@ -1,100 +1,106 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-awslogs
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.2
4
+ version: 1.1.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Anton Klyba
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-12-12 00:00:00.000000000 Z
11
+ date: 2024-01-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: logstash-core-plugin-api
15
14
  requirement: !ruby/object:Gem::Requirement
16
15
  requirements:
17
- - - "~>"
16
+ - - ">="
18
17
  - !ruby/object:Gem::Version
19
- version: '2.0'
20
- type: :runtime
18
+ version: 2.1.12
19
+ - - "<="
20
+ - !ruby/object:Gem::Version
21
+ version: '2.99'
22
+ name: logstash-core-plugin-api
21
23
  prerelease: false
24
+ type: :runtime
22
25
  version_requirements: !ruby/object:Gem::Requirement
23
26
  requirements:
24
- - - "~>"
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: 2.1.12
30
+ - - "<="
25
31
  - !ruby/object:Gem::Version
26
- version: '2.0'
32
+ version: '2.99'
27
33
  - !ruby/object:Gem::Dependency
28
- name: logstash-codec-plain
29
34
  requirement: !ruby/object:Gem::Requirement
30
35
  requirements:
31
36
  - - ">="
32
37
  - !ruby/object:Gem::Version
33
38
  version: '0'
34
- type: :runtime
39
+ name: logstash-codec-plain
35
40
  prerelease: false
41
+ type: :runtime
36
42
  version_requirements: !ruby/object:Gem::Requirement
37
43
  requirements:
38
44
  - - ">="
39
45
  - !ruby/object:Gem::Version
40
46
  version: '0'
41
47
  - !ruby/object:Gem::Dependency
42
- name: aws-sdk-cloudwatchlogs
43
48
  requirement: !ruby/object:Gem::Requirement
44
49
  requirements:
45
50
  - - "~>"
46
51
  - !ruby/object:Gem::Version
47
52
  version: '1'
48
- type: :runtime
53
+ name: aws-sdk-cloudwatchlogs
49
54
  prerelease: false
55
+ type: :runtime
50
56
  version_requirements: !ruby/object:Gem::Requirement
51
57
  requirements:
52
58
  - - "~>"
53
59
  - !ruby/object:Gem::Version
54
60
  version: '1'
55
61
  - !ruby/object:Gem::Dependency
56
- name: logstash-integration-aws
57
62
  requirement: !ruby/object:Gem::Requirement
58
63
  requirements:
59
64
  - - ">="
60
65
  - !ruby/object:Gem::Version
61
66
  version: '0'
62
- type: :runtime
67
+ name: logstash-integration-aws
63
68
  prerelease: false
69
+ type: :runtime
64
70
  version_requirements: !ruby/object:Gem::Requirement
65
71
  requirements:
66
72
  - - ">="
67
73
  - !ruby/object:Gem::Version
68
74
  version: '0'
69
75
  - !ruby/object:Gem::Dependency
70
- name: json
71
76
  requirement: !ruby/object:Gem::Requirement
72
77
  requirements:
73
78
  - - ">="
74
79
  - !ruby/object:Gem::Version
75
80
  version: '0'
76
- type: :runtime
81
+ name: json
77
82
  prerelease: false
83
+ type: :runtime
78
84
  version_requirements: !ruby/object:Gem::Requirement
79
85
  requirements:
80
86
  - - ">="
81
87
  - !ruby/object:Gem::Version
82
88
  version: '0'
83
89
  - !ruby/object:Gem::Dependency
84
- name: logstash-devutils
85
90
  requirement: !ruby/object:Gem::Requirement
86
91
  requirements:
87
92
  - - ">="
88
93
  - !ruby/object:Gem::Version
89
94
  version: '0'
90
- type: :development
95
+ name: logstash-devutils
91
96
  prerelease: false
97
+ type: :development
92
98
  version_requirements: !ruby/object:Gem::Requirement
93
99
  requirements:
94
100
  - - ">="
95
101
  - !ruby/object:Gem::Version
96
102
  version: '0'
97
- description:
103
+ description:
98
104
  email: anarhyst266+gems@gmail.com
99
105
  executables: []
100
106
  extensions: []
@@ -114,7 +120,7 @@ licenses:
114
120
  metadata:
115
121
  logstash_plugin: 'true'
116
122
  logstash_group: output
117
- post_install_message:
123
+ post_install_message:
118
124
  rdoc_options: []
119
125
  require_paths:
120
126
  - lib
@@ -129,8 +135,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
129
135
  - !ruby/object:Gem::Version
130
136
  version: '0'
131
137
  requirements: []
132
- rubygems_version: 3.0.3.1
133
- signing_key:
138
+ rubygems_version: 3.3.26
139
+ signing_key:
134
140
  specification_version: 4
135
141
  summary: Writes events to AWS CloudWatch logs.
136
142
  test_files: []