logstash-output-awslogs 1.1.2 → 1.1.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (5) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +4 -0
  3. data/lib/logstash/outputs/awslogs.rb +87 -52
  4. data/logstash-output-awslogs.gemspec +2 -2
  5. metadata +29 -23
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3f29bdc8b680f033bac338ae4613bc72e73403fbc7664c4c8eba78a0f497fdf3
4
- data.tar.gz: 9b1e38924b1a1994ef71b69a4475f01d99e72345182b5aa1e8a0194b84cf98d6
3
+ metadata.gz: cd6de218003a805ad230d143147375621fbf7bbf803a7739582bf39da44dc228
4
+ data.tar.gz: 19a12a32d0a38df507e498783b046f329f727727bef51e770398afaeb1e7f9d8
5
5
  SHA512:
6
- metadata.gz: 9900328521269d53376e645216989c7c01647f26c1290b8dc88edd5732521a0555fadeb00ad0f4b9674c8b5336b6b550bf62715e92e3be5cd4ec3b5142292cca
7
- data.tar.gz: 51d83fb4c44a8850b848dbc4900f96db80c0fa2521b7b9cf4726bfed3cf97ef7cb72d45ea69413d8f54062fc81773789d58c07177349690fa3766f9b92a7c548
6
+ metadata.gz: 789581c0eff8aaf841a805ffd054733e30cc910897bafc2d245e2cb26ae891e9893ce0e836c709c419c874d5187603a0db6ca7dca96f5236be702f00b02f3e0a
7
+ data.tar.gz: f7531d29688b185862a9e4af27bf796ffa21c638d1423fbaa14b9fb24291eaac89bf08239f2109cfea7fe7edfc1ab3b53b769f866f2635a652d8347ca204b197
data/CHANGELOG.md CHANGED
@@ -3,3 +3,7 @@
3
3
 
4
4
  ## 1.1.1
5
5
  - First fully working version.
6
+
7
+ ## 1.1.3
8
+ - Fixed working with big batches
9
+ - Sequence token removed
data/lib/logstash/outputs/awslogs.rb CHANGED
@@ -4,7 +4,6 @@ require 'logstash/outputs/base'
4
4
  require 'logstash/namespace'
5
5
  require 'logstash/plugin_mixins/aws_config'
6
6
  require 'aws-sdk-cloudwatchlogs'
7
- require 'json'
8
7
 
9
8
 
10
9
  class LogStash::Outputs::Awslogs < LogStash::Outputs::Base
@@ -12,75 +11,111 @@ class LogStash::Outputs::Awslogs < LogStash::Outputs::Base
12
11
 
13
12
  config_name 'awslogs'
14
13
  default :codec, 'line'
14
+ concurrency :shared
15
15
 
16
+ PER_EVENT_OVERHEAD = 26
17
+ MAX_BATCH_SIZE = 1024 * 1024
18
+
19
+ # Log group to send event to
16
20
  config :log_group_name, validate: :string, required: true
21
+ # Logs stream to send event to
17
22
  config :log_stream_name, validate: :string, required: true
23
+ # Message to be sent. Fields interpolation supported. By default the whole event will be sent as a json object
24
+ config :message, validate: :string, default: ""
18
25
 
19
26
  public
20
27
  def register
21
28
  @client = Aws::CloudWatchLogs::Client.new(aws_options_hash)
22
- @next_sequence_tokens = {}
23
29
  end # def register
24
30
 
25
31
  public
26
32
  def multi_receive(events)
27
- to_send = {}
33
+ send_batches = form_event_batches(events)
34
+ send_batches.each do |batch|
35
+ put_events(batch)
36
+ end
37
+ end
38
+
39
+ private
40
+ def put_events(batch)
41
+ begin
42
+ @client.put_log_events(
43
+ {
44
+ log_group_name: batch[:log_group],
45
+ log_stream_name: batch[:log_stream],
46
+ log_events: batch[:log_events]
47
+ }
48
+ )
49
+ rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
50
+ @logger.info('AWSLogs: Will create log group/stream and retry')
51
+ begin
52
+ @client.create_log_group({log_group_name: batch[:log_group]})
53
+ rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
54
+ @logger.info("AWSLogs: Log group #{batch[:log_group]} already exists")
55
+ end
56
+ begin
57
+ @client.create_log_stream({log_group_name: batch[:log_group], log_stream_name: batch[:log_stream]})
58
+ rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
59
+ @logger.info("AWSLogs: Log stream #{batch[:log_stream]} already exists")
60
+ end
61
+ retry
62
+ end
63
+ end
28
64
 
29
- events.each do |event|
65
+ private
66
+ def form_event_batches(events_arr)
67
+ batches = []
68
+ events_by_stream_and_group = {}
69
+ # events_by_stream_and_group = {
70
+ # ['group','stream'] => [{timestamp => 1, message => "message1"},{timestamp => 2, message => "mesage2"}],
71
+ # ['group2','stream2'] => [{timestamp => 3, message => "message3"},{timestamp => 4, message => "message4"}]
72
+ # }
73
+ log_events = events_arr.sort_by { |event| event.timestamp.time.to_f}
74
+ log_events.each do |event|
30
75
  event_log_stream_name = event.sprintf(log_stream_name)
31
76
  event_log_group_name = event.sprintf(log_group_name)
32
77
 
33
- next_sequence_token_key = [event_log_group_name, event_log_stream_name]
34
- unless to_send.keys.include? next_sequence_token_key
35
- to_send.store(next_sequence_token_key, [])
78
+ sort_key = [event_log_group_name, event_log_stream_name]
79
+ unless events_by_stream_and_group.keys.include? sort_key
80
+ events_by_stream_and_group.store(sort_key, [])
81
+ end
82
+ if message.empty?
83
+ events_by_stream_and_group[sort_key].push(
84
+ timestamp: (event.timestamp.time.to_f * 1000).to_int,
85
+ message: event.to_hash.sort.to_h.to_json
86
+ )
87
+ else
88
+ events_by_stream_and_group[sort_key].push(
89
+ timestamp: (event.timestamp.time.to_f * 1000).to_int,
90
+ message: event.sprintf(message)
91
+ )
36
92
  end
37
- to_send[next_sequence_token_key].push(
38
- timestamp: (event.timestamp.time.to_f * 1000).to_int,
39
- message: event.to_hash.sort.to_h.to_json
40
- )
41
93
  end
42
-
43
-
44
- to_send.each do |event_log_names, log_events|
45
- event_log_group_name = event_log_names[0]
46
- event_log_stream_name = event_log_names[1]
47
- next_sequence_token_key = [event_log_group_name, event_log_stream_name]
48
-
49
- log_events.sort_by!{ |event| event[:timestamp] }
50
-
51
- ident_opts = {
52
- log_group_name: event_log_group_name,
53
- log_stream_name: event_log_stream_name
94
+ events_by_stream_and_group.each do |key, value|
95
+ temp_batch = {
96
+ log_group: key[0],
97
+ log_stream: key[1],
98
+ size: 0,
99
+ log_events: []
54
100
  }
55
- send_opts = ident_opts.merge(
56
- log_events: log_events
57
- )
58
- if @next_sequence_tokens.keys.include? next_sequence_token_key
59
- send_opts[:sequence_token] = @next_sequence_tokens[next_sequence_token_key]
60
- end
61
- begin
62
- resp = @client.put_log_events(send_opts)
63
- @next_sequence_tokens.store(next_sequence_token_key, resp.next_sequence_token)
64
- rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
65
- @logger.info('Will create log group/stream and retry')
66
- begin
67
- @client.create_log_group({log_group_name: send_opts[:log_group_name]})
68
- rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
69
- @logger.info("Log group #{send_opts[:log_group_name]} already exists")
70
- end
71
- begin
72
- @client.create_log_stream({log_group_name: send_opts[:log_group_name], log_stream_name: send_opts[:log_stream_name]})
73
- rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
74
- @logger.info("Log stream #{send_opts[:log_stream_name]} already exists")
101
+ value.each do |log_event|
102
+ if log_event[:message].bytesize + PER_EVENT_OVERHEAD + temp_batch[:size] < MAX_BATCH_SIZE
103
+ temp_batch[:size] += log_event[:message].bytesize + PER_EVENT_OVERHEAD
104
+ temp_batch[:log_events] << log_event
105
+ else
106
+ batches << temp_batch
107
+ temp_batch = {
108
+ log_group: key[0],
109
+ log_stream: key[1],
110
+ size: 0,
111
+ log_events: []
112
+ }
113
+ temp_batch[:size] += log_event[:message].bytesize + PER_EVENT_OVERHEAD
114
+ temp_batch[:log_events] << log_event
75
115
  end
76
- retry
77
- rescue Aws::CloudWatchLogs::Errors::InvalidSequenceTokenException => e
78
- send_opts[:sequence_token] = e.expected_sequence_token
79
- retry
80
- rescue Aws::CloudWatchLogs::Errors::ThrottlingException
81
- @logger.info('Logs throttling, retry')
82
- retry
83
116
  end
117
+ batches << temp_batch
84
118
  end
85
- end # def multi_receive_encodeds
86
- end # class LogStash::Outputs::Awslogs
119
+ batches
120
+ end
121
+ end
data/logstash-output-awslogs.gemspec CHANGED
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-awslogs'
3
- s.version = '1.1.2'
3
+ s.version = '1.1.3'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = 'Writes events to AWS CloudWatch logs.'
6
6
  s.homepage = 'https://github.com/Anarhyst266/logstash-output-awslogs'
@@ -17,7 +17,7 @@ Gem::Specification.new do |s|
17
17
  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
18
18
 
19
19
  # Gem dependencies
20
- s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
20
+ s.add_runtime_dependency "logstash-core-plugin-api", ">= 2.1.12", "<= 2.99"
21
21
  s.add_runtime_dependency "logstash-codec-plain"
22
22
  s.add_runtime_dependency "aws-sdk-cloudwatchlogs", '~> 1'
23
23
  s.add_runtime_dependency "logstash-integration-aws"
metadata CHANGED
@@ -1,100 +1,106 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-awslogs
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.2
4
+ version: 1.1.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Anton Klyba
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-12-12 00:00:00.000000000 Z
11
+ date: 2023-06-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: logstash-core-plugin-api
15
14
  requirement: !ruby/object:Gem::Requirement
16
15
  requirements:
17
- - - "~>"
16
+ - - ">="
18
17
  - !ruby/object:Gem::Version
19
- version: '2.0'
20
- type: :runtime
18
+ version: 2.1.12
19
+ - - "<="
20
+ - !ruby/object:Gem::Version
21
+ version: '2.99'
22
+ name: logstash-core-plugin-api
21
23
  prerelease: false
24
+ type: :runtime
22
25
  version_requirements: !ruby/object:Gem::Requirement
23
26
  requirements:
24
- - - "~>"
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: 2.1.12
30
+ - - "<="
25
31
  - !ruby/object:Gem::Version
26
- version: '2.0'
32
+ version: '2.99'
27
33
  - !ruby/object:Gem::Dependency
28
- name: logstash-codec-plain
29
34
  requirement: !ruby/object:Gem::Requirement
30
35
  requirements:
31
36
  - - ">="
32
37
  - !ruby/object:Gem::Version
33
38
  version: '0'
34
- type: :runtime
39
+ name: logstash-codec-plain
35
40
  prerelease: false
41
+ type: :runtime
36
42
  version_requirements: !ruby/object:Gem::Requirement
37
43
  requirements:
38
44
  - - ">="
39
45
  - !ruby/object:Gem::Version
40
46
  version: '0'
41
47
  - !ruby/object:Gem::Dependency
42
- name: aws-sdk-cloudwatchlogs
43
48
  requirement: !ruby/object:Gem::Requirement
44
49
  requirements:
45
50
  - - "~>"
46
51
  - !ruby/object:Gem::Version
47
52
  version: '1'
48
- type: :runtime
53
+ name: aws-sdk-cloudwatchlogs
49
54
  prerelease: false
55
+ type: :runtime
50
56
  version_requirements: !ruby/object:Gem::Requirement
51
57
  requirements:
52
58
  - - "~>"
53
59
  - !ruby/object:Gem::Version
54
60
  version: '1'
55
61
  - !ruby/object:Gem::Dependency
56
- name: logstash-integration-aws
57
62
  requirement: !ruby/object:Gem::Requirement
58
63
  requirements:
59
64
  - - ">="
60
65
  - !ruby/object:Gem::Version
61
66
  version: '0'
62
- type: :runtime
67
+ name: logstash-integration-aws
63
68
  prerelease: false
69
+ type: :runtime
64
70
  version_requirements: !ruby/object:Gem::Requirement
65
71
  requirements:
66
72
  - - ">="
67
73
  - !ruby/object:Gem::Version
68
74
  version: '0'
69
75
  - !ruby/object:Gem::Dependency
70
- name: json
71
76
  requirement: !ruby/object:Gem::Requirement
72
77
  requirements:
73
78
  - - ">="
74
79
  - !ruby/object:Gem::Version
75
80
  version: '0'
76
- type: :runtime
81
+ name: json
77
82
  prerelease: false
83
+ type: :runtime
78
84
  version_requirements: !ruby/object:Gem::Requirement
79
85
  requirements:
80
86
  - - ">="
81
87
  - !ruby/object:Gem::Version
82
88
  version: '0'
83
89
  - !ruby/object:Gem::Dependency
84
- name: logstash-devutils
85
90
  requirement: !ruby/object:Gem::Requirement
86
91
  requirements:
87
92
  - - ">="
88
93
  - !ruby/object:Gem::Version
89
94
  version: '0'
90
- type: :development
95
+ name: logstash-devutils
91
96
  prerelease: false
97
+ type: :development
92
98
  version_requirements: !ruby/object:Gem::Requirement
93
99
  requirements:
94
100
  - - ">="
95
101
  - !ruby/object:Gem::Version
96
102
  version: '0'
97
- description:
103
+ description:
98
104
  email: anarhyst266+gems@gmail.com
99
105
  executables: []
100
106
  extensions: []
@@ -114,7 +120,7 @@ licenses:
114
120
  metadata:
115
121
  logstash_plugin: 'true'
116
122
  logstash_group: output
117
- post_install_message:
123
+ post_install_message:
118
124
  rdoc_options: []
119
125
  require_paths:
120
126
  - lib
@@ -129,8 +135,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
129
135
  - !ruby/object:Gem::Version
130
136
  version: '0'
131
137
  requirements: []
132
- rubygems_version: 3.0.3.1
133
- signing_key:
138
+ rubygems_version: 3.3.26
139
+ signing_key:
134
140
  specification_version: 4
135
141
  summary: Writes events to AWS CloudWatch logs.
136
142
  test_files: []