logstash-integration-aws 0.1.0.pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.PRE.MERGE.md +658 -0
  3. data/CHANGELOG.md +15 -0
  4. data/CONTRIBUTORS +40 -0
  5. data/Gemfile +11 -0
  6. data/LICENSE +202 -0
  7. data/NOTICE.TXT +5 -0
  8. data/README.md +205 -0
  9. data/docs/codec-cloudfront.asciidoc +53 -0
  10. data/docs/codec-cloudtrail.asciidoc +45 -0
  11. data/docs/index.asciidoc +38 -0
  12. data/docs/input-cloudwatch.asciidoc +320 -0
  13. data/docs/input-s3.asciidoc +346 -0
  14. data/docs/input-sqs.asciidoc +287 -0
  15. data/docs/output-cloudwatch.asciidoc +321 -0
  16. data/docs/output-s3.asciidoc +442 -0
  17. data/docs/output-sns.asciidoc +166 -0
  18. data/docs/output-sqs.asciidoc +242 -0
  19. data/lib/logstash/codecs/cloudfront.rb +84 -0
  20. data/lib/logstash/codecs/cloudtrail.rb +47 -0
  21. data/lib/logstash/inputs/cloudwatch.rb +338 -0
  22. data/lib/logstash/inputs/s3.rb +466 -0
  23. data/lib/logstash/inputs/sqs.rb +196 -0
  24. data/lib/logstash/outputs/cloudwatch.rb +346 -0
  25. data/lib/logstash/outputs/s3/file_repository.rb +121 -0
  26. data/lib/logstash/outputs/s3/path_validator.rb +18 -0
  27. data/lib/logstash/outputs/s3/size_and_time_rotation_policy.rb +24 -0
  28. data/lib/logstash/outputs/s3/size_rotation_policy.rb +26 -0
  29. data/lib/logstash/outputs/s3/temporary_file.rb +71 -0
  30. data/lib/logstash/outputs/s3/temporary_file_factory.rb +129 -0
  31. data/lib/logstash/outputs/s3/time_rotation_policy.rb +26 -0
  32. data/lib/logstash/outputs/s3/uploader.rb +74 -0
  33. data/lib/logstash/outputs/s3/writable_directory_validator.rb +17 -0
  34. data/lib/logstash/outputs/s3/write_bucket_permission_validator.rb +60 -0
  35. data/lib/logstash/outputs/s3.rb +405 -0
  36. data/lib/logstash/outputs/sns.rb +133 -0
  37. data/lib/logstash/outputs/sqs.rb +167 -0
  38. data/lib/logstash/plugin_mixins/aws_config/generic.rb +54 -0
  39. data/lib/logstash/plugin_mixins/aws_config/v2.rb +93 -0
  40. data/lib/logstash/plugin_mixins/aws_config.rb +8 -0
  41. data/logstash-integration-aws.gemspec +52 -0
  42. data/spec/codecs/cloudfront_spec.rb +92 -0
  43. data/spec/codecs/cloudtrail_spec.rb +56 -0
  44. data/spec/fixtures/aws_credentials_file_sample_test.yml +2 -0
  45. data/spec/fixtures/aws_temporary_credentials_file_sample_test.yml +3 -0
  46. data/spec/fixtures/cloudfront.log +4 -0
  47. data/spec/fixtures/compressed.log.gee.zip +0 -0
  48. data/spec/fixtures/compressed.log.gz +0 -0
  49. data/spec/fixtures/compressed.log.gzip +0 -0
  50. data/spec/fixtures/invalid_utf8.gbk.log +2 -0
  51. data/spec/fixtures/json.log +2 -0
  52. data/spec/fixtures/json_with_message.log +2 -0
  53. data/spec/fixtures/multiline.log +6 -0
  54. data/spec/fixtures/multiple_compressed_streams.gz +0 -0
  55. data/spec/fixtures/uncompressed.log +2 -0
  56. data/spec/inputs/cloudwatch_spec.rb +85 -0
  57. data/spec/inputs/s3_spec.rb +610 -0
  58. data/spec/inputs/sincedb_spec.rb +17 -0
  59. data/spec/inputs/sqs_spec.rb +324 -0
  60. data/spec/integration/cloudwatch_spec.rb +25 -0
  61. data/spec/integration/dynamic_prefix_spec.rb +92 -0
  62. data/spec/integration/gzip_file_spec.rb +62 -0
  63. data/spec/integration/gzip_size_rotation_spec.rb +63 -0
  64. data/spec/integration/outputs/sqs_spec.rb +98 -0
  65. data/spec/integration/restore_from_crash_spec.rb +67 -0
  66. data/spec/integration/s3_spec.rb +66 -0
  67. data/spec/integration/size_rotation_spec.rb +59 -0
  68. data/spec/integration/sqs_spec.rb +110 -0
  69. data/spec/integration/stress_test_spec.rb +60 -0
  70. data/spec/integration/time_based_rotation_with_constant_write_spec.rb +60 -0
  71. data/spec/integration/time_based_rotation_with_stale_write_spec.rb +64 -0
  72. data/spec/integration/upload_current_file_on_shutdown_spec.rb +51 -0
  73. data/spec/outputs/cloudwatch_spec.rb +38 -0
  74. data/spec/outputs/s3/file_repository_spec.rb +143 -0
  75. data/spec/outputs/s3/size_and_time_rotation_policy_spec.rb +77 -0
  76. data/spec/outputs/s3/size_rotation_policy_spec.rb +41 -0
  77. data/spec/outputs/s3/temporary_file_factory_spec.rb +89 -0
  78. data/spec/outputs/s3/temporary_file_spec.rb +47 -0
  79. data/spec/outputs/s3/time_rotation_policy_spec.rb +60 -0
  80. data/spec/outputs/s3/uploader_spec.rb +69 -0
  81. data/spec/outputs/s3/writable_directory_validator_spec.rb +40 -0
  82. data/spec/outputs/s3/write_bucket_permission_validator_spec.rb +49 -0
  83. data/spec/outputs/s3_spec.rb +232 -0
  84. data/spec/outputs/sns_spec.rb +160 -0
  85. data/spec/plugin_mixin/aws_config_spec.rb +217 -0
  86. data/spec/spec_helper.rb +8 -0
  87. data/spec/support/helpers.rb +119 -0
  88. data/spec/unit/outputs/sqs_spec.rb +247 -0
  89. metadata +467 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 9aed616c6b3abd22f2196b4939ffbedbf1d2c8785a5aef119fc9fe8242847921
4
+ data.tar.gz: e80bfc1603c49e9271111c8ff4767e42ddcfaa8dedbca2100e3b9e76ae10d735
5
+ SHA512:
6
+ metadata.gz: 5a9c24ffdc56ec13be5b32e27016fc55d4a4a256b16f6d8f28fa38128b1cb1433abd19e63b245ab6d693539af612ba29ad0f345c5a28e96c64a52d66f867c778
7
+ data.tar.gz: b14a38aea9eafb67c3b0d516463c43540f807f5f3b63fe459841023becd7003ebdb188f202dc7b3233c40b995c3a13ca41a1bcdf31e55abc124eac0954ff0da9
@@ -0,0 +1,658 @@
1
+ # CHANGELOG - logstash-codec-cloudfront
2
+
3
+ ## 3.0.3
4
+ - Update gemspec summary
5
+
6
+ ## 3.0.2
7
+ - Fix some documentation issues
8
+
9
+ # 2.0.4
10
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
11
+ # 2.0.3
12
+ - New dependency requirements for logstash-core for the 5.0 release
13
+ ## 2.0.0
14
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
15
+ instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
16
+ - Dependency on logstash-core update to 2.0
17
+
18
+ ---------
19
+
20
+ # CHANGELOG - logstash-codec-cloudtrail
21
+
22
+ ## 3.0.5
23
+ - [#22](https://github.com/logstash-plugins/logstash-codec-cloudtrail/pull/22)Handle 'sourceIpAddress' fields with non-ip address content by moving them to 'sourceHost' field
24
+
25
+ ## 3.0.4
26
+ - Don't crash when data doesn't contain some particular elements
27
+
28
+ ## 3.0.3
29
+ - Fix some documentation issues
30
+
31
+ # 3.0.1
32
+ - fixed mapping template for requestParameters.disableApiTermination field
33
+
34
+ ## 3.0.0
35
+ - Update to support Logstash 2.4 & 5.0 APIs
36
+
37
+ ## 2.0.4
38
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
39
+
40
+ ## 2.0.3
41
+ - New dependency requirements for logstash-core for the 5.0 release
42
+
43
+ ## 2.0.0
44
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
45
+ instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
46
+ - Dependency on logstash-core update to 2.0
47
+
48
+ ---------
49
+
50
+ # CHANGELOG - logstash-input-cloudwatch
51
+
52
+ ## 2.2.4
53
+ - Fixed shutdown handling [#43](https://github.com/logstash-plugins/logstash-input-cloudwatch/pull/43)
54
+
55
+ ## 2.2.3
56
+ - Fixed issue where metric timestamp was being lost due to over-writing by end_time [#38](https://github.com/logstash-plugins/logstash-input-cloudwatch/pull/38)
57
+
58
+ ## 2.2.2
59
+ - Added ability to use AWS/EC2 namespace without requiring filters [#36](https://github.com/logstash-plugins/logstash-input-cloudwatch/pull/36)
60
+
61
+ ## 2.2.1
62
+ - Fixed README.md link to request metric support to point to this repo [#34](https://github.com/logstash-plugins/logstash-input-cloudwatch/pull/34)
63
+
64
+ ## 2.2.0
65
+ - Changed to use the underlying version of the AWS SDK to v2. [#32](https://github.com/logstash-plugins/logstash-input-cloudwatch/pull/32)
66
+ - Fixed License definition in gemspec to be valid SPDX identifier [#32](https://github.com/logstash-plugins/logstash-input-cloudwatch/pull/32)
67
+ - Fixed fatal error when using secret key attribute in config [#30](https://github.com/logstash-plugins/logstash-input-cloudwatch/issues/30)
68
+
69
+ ## 2.1.1
70
+ - Docs: Set the default_codec doc attribute.
71
+
72
+ ## 2.1.0
73
+ - Add documentation for endpoint, role_arn and role_session_name #29
74
+ - Reduce info level logging verbosity #27
75
+
76
+ ## 2.0.3
77
+ - Update gemspec summary
78
+
79
+ ## 2.0.2
80
+ - Fix some documentation issues
81
+
82
+ # 1.1.3
83
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
84
+ # 1.1.1
85
+ - New dependency requirements for logstash-core for the 5.0 release
86
+ ## 1.1.0
87
+ - Moved from jrgns/logstash-input-cloudwatch to logstash-plugins
88
+
89
+ ---------
90
+
91
+ # CHANGELOG - logstash-input-s3
92
+
93
+ ## 3.8.4
94
+ - Refactoring, reuse code to manage `additional_settings` from mixin-aws [#237](https://github.com/logstash-plugins/logstash-input-s3/pull/237)
95
+
96
+ ## 3.8.3
97
+ - Fix missing `metadata` and `type` of the last event [#223](https://github.com/logstash-plugins/logstash-input-s3/pull/223)
98
+
99
+ ## 3.8.2
100
+ - Refactor: read sincedb time once per bucket listing [#233](https://github.com/logstash-plugins/logstash-input-s3/pull/233)
101
+
102
+ ## 3.8.1
103
+ - Feat: cast true/false values for additional_settings [#232](https://github.com/logstash-plugins/logstash-input-s3/pull/232)
104
+
105
+ ## 3.8.0
106
+ - Add ECS v8 support.
107
+
108
+ ## 3.7.0
109
+ - Add ECS support. [#228](https://github.com/logstash-plugins/logstash-input-s3/pull/228)
110
+ - Fix missing file in cutoff time change. [#224](https://github.com/logstash-plugins/logstash-input-s3/pull/224)
111
+
112
+ ## 3.6.0
113
+ - Fixed unprocessed file with the same `last_modified` in ingestion. [#220](https://github.com/logstash-plugins/logstash-input-s3/pull/220)
114
+
115
+ ## 3.5.2
116
+ - [DOC]Added note that only AWS S3 is supported. No other S3 compatible storage solutions are supported. [#208](https://github.com/logstash-plugins/logstash-input-s3/issues/208)
117
+
118
+ ## 3.5.1
119
+ - [DOC]Added example for `exclude_pattern` and reordered option descriptions [#204](https://github.com/logstash-plugins/logstash-input-s3/issues/204)
120
+
121
+ ## 3.5.0
122
+ - Added support for including objects restored from Glacier or Glacier Deep [#199](https://github.com/logstash-plugins/logstash-input-s3/issues/199)
123
+ - Added `gzip_pattern` option, enabling more flexible determination of whether a file is gzipped [#165](https://github.com/logstash-plugins/logstash-input-s3/issues/165)
124
+ - Refactor: log exception: class + unify logging messages a bit [#201](https://github.com/logstash-plugins/logstash-input-s3/pull/201)
125
+
126
+ ## 3.4.1
127
+ - Fixed link formatting for input type (documentation)
128
+
129
+ ## 3.4.0
130
+ - Skips objects that are archived to AWS Glacier with a helpful log message (previously they would log as matched, but then fail to load events) [#160](https://github.com/logstash-plugins/logstash-input-s3/pull/160)
131
+ - Added `watch_for_new_files` option, enabling single-batch imports [#159](https://github.com/logstash-plugins/logstash-input-s3/pull/159)
132
+
133
+ ## 3.3.7
134
+ - Added ability to optionally include S3 object properties inside @metadata [#155](https://github.com/logstash-plugins/logstash-input-s3/pull/155)
135
+
136
+ ## 3.3.6
137
+ - Fixed error in documentation by removing illegal commas [#154](https://github.com/logstash-plugins/logstash-input-s3/pull/154)
138
+
139
+ ## 3.3.5
140
+ - [#136](https://github.com/logstash-plugins/logstash-input-s3/pull/136) Avoid plugin crashes when encountering 'bad' files in S3 buckets
141
+
142
+ ## 3.3.4
143
+ - Log entry when bucket is empty #150
144
+
145
+ ## 3.3.3
146
+ - Symbolize hash keys for additional_settings hash #148
147
+
148
+ ## 3.3.2
149
+ - Docs: Set the default_codec doc attribute.
150
+
151
+ ## 3.3.1
152
+ - Improve error handling when listing/downloading from S3 #144
153
+
154
+ ## 3.3.0
155
+ - Add documentation for endpoint, role_arn and role_session_name #142
156
+ - Add support for additional_settings option #141
157
+
158
+ ## 3.2.0
159
+ - Add support for auto-detecting gzip files with `.gzip` extension, in addition to existing support for `*.gz`
160
+ - Improve performance of gzip decoding by 10x by using Java's Zlib
161
+
162
+ ## 3.1.9
163
+ - Change default sincedb path to live in `{path.data}/plugins/inputs/s3` instead of $HOME.
164
+ Prior Logstash installations (using $HOME default) are automatically migrated.
165
+ - Don't download the file if the length is 0 #2
166
+
167
+ ## 3.1.8
168
+ - Update gemspec summary
169
+
170
+ ## 3.1.7
171
+ - Fix missing last multi-line entry #120
172
+
173
+ ## 3.1.6
174
+ - Fix some documentation issues
175
+
176
+ ## 3.1.4
177
+ - Avoid parsing non string elements #109
178
+
179
+ ## 3.1.3
180
+ - The plugin will now include the s3 key in the metadata #105
181
+
182
+ ## 3.1.2
183
+ - Fix an issue when the remote file contains multiple blob of gz in the same file #101
184
+ - Make the integration suite run
185
+ - Remove uneeded development dependency
186
+
187
+ ## 3.1.1
188
+ - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
189
+
190
+ ## 3.1.0
191
+ - breaking,config: Remove deprecated config `credentials` and `region_endpoint`. Please use AWS mixin.
192
+
193
+ ## 3.0.1
194
+ - Republish all the gems under jruby.
195
+
196
+ ## 3.0.0
197
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
198
+
199
+ ## 2.0.6
200
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
201
+
202
+ ## 2.0.5
203
+ - New dependency requirements for logstash-core for the 5.0 release
204
+
205
+ ## 2.0.4
206
+ - Fix for Error: No Such Key problem when deleting
207
+
208
+ ## 2.0.3
209
+ - Do not raise an exception if the sincedb file is empty, instead return the current time #66
210
+
211
+ ## 2.0.0
212
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
213
+ instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
214
+ - Dependency on logstash-core update to 2.0
215
+
216
+ ---------
217
+
218
+ # CHANGELOG - logstash-input-sqs
219
+
220
+ ## 3.3.2
221
+ - Fix an issue that prevented timely shutdown when subscribed to an inactive queue [#65](https://github.com/logstash-plugins/logstash-input-sqs/pull/65)
222
+
223
+ ## 3.3.1
224
+ - Refactoring: used logstash-mixin-aws to leverage shared code to manage `additional_settings` [#64](https://github.com/logstash-plugins/logstash-input-sqs/pull/64)
225
+
226
+ ## 3.3.0
227
+ - Feature: Add `additional_settings` option to fine-grain configuration of AWS client [#61](https://github.com/logstash-plugins/logstash-input-sqs/pull/61)
228
+
229
+ ## 3.2.0
230
+ - Feature: Add `queue_owner_aws_account_id` parameter for cross-account queues [#60](https://github.com/logstash-plugins/logstash-input-sqs/pull/60)
231
+
232
+ ## 3.1.3
233
+ - Fix: retry networking errors (with backoff) [#57](https://github.com/logstash-plugins/logstash-input-sqs/pull/57)
234
+
235
+ ## 3.1.2
236
+ - Added support for multiple events inside same message from SQS [#48](https://github.com/logstash-plugins/logstash-input-sqs/pull/48/files)
237
+
238
+ ## 3.1.1
239
+ - Docs: Set the default_codec doc attribute.
240
+
241
+ ## 3.1.0
242
+ - Add documentation for endpoint, role_arn and role_session_name #46
243
+ - Fix sample IAM policy to match to match the documentation #32
244
+
245
+ ## 3.0.6
246
+ - Update gemspec summary
247
+
248
+ ## 3.0.5
249
+ - Fix some documentation issues
250
+
251
+ ## 3.0.3
252
+ - Monkey-patch the AWS-SDK to prevent "uninitialized constant" errors.
253
+
254
+ ## 3.0.2
255
+ - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
256
+
257
+ ## 3.0.1
258
+ - Republish all the gems under jruby.
259
+ ## 3.0.0
260
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
261
+ # 2.0.5
262
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
263
+ # 2.0.4
264
+ - New dependency requirements for logstash-core for the 5.0 release
265
+ ## 2.0.3
266
+ - Fixes #22, wrong key use on the stats object
267
+ ## 2.0.0
268
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
269
+ instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
270
+ - Dependency on logstash-core update to 2.0
271
+
272
+ # 1.1.0
273
+ - AWS ruby SDK v2 upgrade
274
+ - Replaces aws-sdk dependencies with mixin-aws
275
+ - Removes unnecessary de-allocation
276
+ - Move the code into smaller methods to allow easier mocking and testing
277
+ - Add the option to configure polling frequency
278
+ - Adding a monkey patch to make sure `LogStash::ShutdownSignal` doesn't get catch by AWS RetryError.
279
+
280
+ ---------
281
+
282
+ # CHANGELOG - logstash-mixin-aws
283
+
284
+ ## 5.1.0
285
+ - Add support for 'addition_settings' configuration options used by S3 and SQS input plugins [#53](https://github.com/logstash-plugins/logstash-mixin-aws/pull/53).
286
+
287
+ ## 5.0.0
288
+ - Drop support for aws-sdk-v1
289
+
290
+ ## 4.4.1
291
+ - Fix: proxy with assumed role (properly) [#50](https://github.com/logstash-plugins/logstash-mixin-aws/pull/50).
292
+
293
+ ## 4.4.0
294
+ - Fix: credentials/proxy with assumed role [#48](https://github.com/logstash-plugins/logstash-mixin-aws/pull/48).
295
+ Plugin no longer assumes `access_key_id`/`secret_access_key` credentials not to be set when `role_arn` specified.
296
+
297
+ ## 4.3.0
298
+ - Drop strict value validation for region option #36
299
+ - Add endpoint option to customize the endpoint uri #32
300
+ - Allow user to provide a role to assume #27
301
+ - Update aws-sdk dependency to '~> 2'
302
+
303
+ ## 4.2.4
304
+ - Minor config validation fixes
305
+
306
+ ## 4.2.3
307
+ - Fix some documentation issues
308
+
309
+ ## 4.2.1
310
+ - Add eu-west-2, us-east-2 and ca-central-1 regions
311
+
312
+ ## 4.2.0
313
+ - Add region ap-south-1
314
+
315
+ ## 4.1.0
316
+ - Update aws-sdk to ~> 2.3.0
317
+
318
+ ## 4.0.2
319
+ - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
320
+
321
+ ## 4.0.1
322
+ - Republish all the gems under jruby.
323
+ ## 4.0.0
324
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
325
+ # 2.0.4
326
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
327
+ # 2.0.3
328
+ - New dependency requirements for logstash-core for the 5.0 release
329
+ ## 2.0.0
330
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
331
+ instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
332
+ - Dependency on logstash-core update to 2.0
333
+
334
+ # 1.0.1
335
+ * Correctly set proxy options on V2 of the aws-sdk
336
+
337
+ # 1.0.0
338
+ * Allow to use either V1 or V2 of the `AWS-SDK` in your plugins. Fixes: https://github.com/logstash-plugins/logstash-mixin-aws/issues/8
339
+
340
+ ---------
341
+
342
+ # CHANGELOG - logstash-output-cloudwatch
343
+
344
+ ## 3.0.10
345
+ - Deps: unpin rufus scheduler [#20](https://github.com/logstash-plugins/logstash-output-cloudwatch/pull/20)
346
+ - Fix: an old undefined method error which would surface with load (as queue fills up)
347
+
348
+ ## 3.0.9
349
+ - Fix: dropped usage of SHUTDOWN event deprecated since Logstash 5.0 [#18](https://github.com/logstash-plugins/logstash-output-cloudwatch/pull/18)
350
+
351
+ ## 3.0.8
352
+ - Docs: Set the default_codec doc attribute.
353
+
354
+ ## 3.0.7
355
+ - Update gemspec summary
356
+
357
+ ## 3.0.6
358
+ - Fix some documentation issues
359
+
360
+ ## 3.0.4
361
+ - Fix some remaining uses of the old event api. blocking the use of this plugin
362
+
363
+ ## 3.0.3
364
+ - Move some log messages from info to debug to avoid noise
365
+
366
+ ## 3.0.2
367
+ - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
368
+
369
+ ## 3.0.1
370
+ - Republish all the gems under jruby.
371
+ ## 3.0.0
372
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
373
+ # 2.0.4
374
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
375
+ # 2.0.3
376
+ - New dependency requirements for logstash-core for the 5.0 release
377
+ ## 2.0.0
378
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
379
+ instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
380
+ - Dependency on logstash-core update to 2.0
381
+
382
+ ---------
383
+
384
+ # CHANGELOG - logstash-output-sns
385
+
386
+ ## 4.0.8
387
+ - Feat: handle host object as subject (due ECS) [#22](https://github.com/logstash-plugins/logstash-output-sns/pull/22)
388
+
389
+ ## 4.0.7
390
+ - Docs: Set the default_codec doc attribute.
391
+
392
+ ## 4.0.6
393
+ - Update gemspec summary
394
+
395
+ ## 4.0.5
396
+ - Fix some documentation issues
397
+
398
+ ## 4.0.3
399
+ - Mark this output as thread safe to allow concurrent connections to AWS.
400
+
401
+ ## 4.0.2
402
+ - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
403
+
404
+ ## 4.0.1
405
+ - Republish all the gems under jruby.
406
+ ## 4.0.0
407
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
408
+ # 3.0.4
409
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
410
+ # 3.0.3
411
+ - New dependency requirements for logstash-core for the 5.0 release
412
+ ## 3.0.0
413
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
414
+ instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
415
+ - Dependency on logstash-core update to 2.0
416
+
417
+ # 1.0.1
418
+ * Properly trim messages for AWS without breaking unicode byte boundaries
419
+
420
+ # 1.0.0
421
+ * Full refactor.
422
+ * This plugin now uses codecs for all formatting. The 'format' option has now been removed. Please use a codec.
423
+ # 0.1.5
424
+ * If no `subject` are specified fallback to the %{host} key (https://github.com/logstash-plugins/logstash-output-sns/pull/2)
425
+ * Migrate the SNS Api to use the AWS-SDK v2
426
+
427
+ ---------
428
+
429
+ # CHANGELOG - logstash-output-sqs
430
+
431
+ ## 6.0.1
432
+ - Added missing index entry for `queue_owner_aws_account_id` [#33](https://github.com/logstash-plugins/logstash-output-sqs/pull/33)
433
+
434
+ ## 6.0.0
435
+ - Removed obsolete fields `batch` and `batch_timeout`
436
+ - Removed workaround to JRuby bug (see more [here](https://github.com/jruby/jruby/issues/3645))
437
+
438
+ ## 5.1.2
439
+ - Added the ability to send to a different account id's queue. [#30](https://github.com/logstash-plugins/logstash-output-sqs/pull/30)
440
+
441
+ ## 5.1.1
442
+ - Docs: Set the default_codec doc attribute.
443
+
444
+ ## 5.1.0
445
+ - Add documentation for endpoint, role_arn and role_session_name #29
446
+
447
+ ## 5.0.2
448
+ - Update gemspec summary
449
+
450
+ ## 5.0.1
451
+ - Fix some documentation issues
452
+
453
+ ## 5.0.0
454
+ - Breaking: mark deprecated `batch` and `batch_timeout` options as obsolete
455
+
456
+ ## 4.0.1
457
+ - Docs: Fix doc generation issue by removing extraneous comments.
458
+
459
+ ## 4.0.0
460
+ - Add unit and integration tests.
461
+ - Adjust the sample IAM policy in the documentation, removing actions which are not actually required by the plugin. Specifically, the following actions are not required: `sqs:ChangeMessageVisibility`, `sqs:ChangeMessageVisibilityBatch`, `sqs:GetQueueAttributes` and `sqs:ListQueues`.
462
+ - Dynamically adjust the batch message size. SQS allows up to 10 messages to be published in a single batch, however the total size of the batch is limited to 256KiB (see [Limits in Amazon SQS](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/limits-messages.html)). This plugin will now dynamically adjust the number of events included in each batch to ensure that the total batch size does not exceed `message_max_size`. Note that any single messages which exceeds the 256KiB size limit will be dropped.
463
+ - Move to the new concurrency model, `:shared`.
464
+ - The `batch_timeout` parameter has been deprecated because it no longer has any effect.
465
+ - The individual (non-batch) mode of operation (i.e. `batch => false`) has been deprecated. Batch mode is vastly more performant and we do not believe that there are any use cases which require non-batch mode. You can emulate non-batch mode by setting `batch_events => 1`, although this will call `sqs:SendMessageBatch` with a batch size of 1 rather than calling `sqs:SendMessage`.
466
+ - The plugin now implements `#multi_receive_encoded` and no longer uses `Stud::Buffer`.
467
+ - Update the AWS SDK to version 2.
468
+
469
+ ## 3.0.2
470
+ - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
471
+
472
+ ## 3.0.1
473
+ - Republish all the gems under jruby.
474
+ ## 3.0.0
475
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
476
+ # 2.0.5
477
+ - Add travis config and build status
478
+ - Require the AWS mixin to be higher than 1.0.0
479
+ # 2.0.4
480
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
481
+ # 2.0.3
482
+ - New dependency requirements for logstash-core for the 5.0 release
483
+ ## 2.0.0
484
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
485
+ instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
486
+ - Dependency on logstash-core update to 2.0
487
+
488
+ # CHANGELOG - logstash - output-s3
489
+
490
+ ## 4.3.5
491
+ - Feat: cast true/false values for additional_settings [#241](https://github.com/logstash-plugins/logstash-output-s3/pull/241)
492
+
493
+ ## 4.3.4
494
+ - [DOC] Added note about performance implications of interpolated strings in prefixes [#233](https://github.com/logstash-plugins/logstash-output-s3/pull/233)
495
+
496
+ ## 4.3.3
497
+ - [DOC] Updated links to use shared attributes [#230](https://github.com/logstash-plugins/logstash-output-s3/pull/230)
498
+
499
+ ## 4.3.2
500
+ - [DOC] Added note that only AWS S3 is supported. No other S3 compatible storage solutions are supported. [#223](https://github.com/logstash-plugins/logstash-output-s3/pull/223)
501
+
502
+ ## 4.3.1
503
+ - [DOC] Updated setting descriptions for clarity [#219](https://github.com/logstash-plugins/logstash-output-s3/pull/219) and [#220](https://github.com/logstash-plugins/logstash-output-s3/pull/220)
504
+
505
+ ## 4.3.0
506
+ - Feat: Added retry_count and retry_delay config [#218](https://github.com/logstash-plugins/logstash-output-s3/pull/218)
507
+
508
+ ## 4.2.0
509
+ - Added ability to specify [ONEZONE_IA](https://aws.amazon.com/s3/storage-classes/#__) as storage_class
510
+
511
+ ## 4.1.10
512
+ - Added clarification for endpoint in documentation [#198](https://github.com/logstash-plugins/logstash-output-s3/pull/198)
513
+
514
+ ## 4.1.9
515
+ - Added configuration information for multiple s3 outputs to documentation [#196](https://github.com/logstash-plugins/logstash-output-s3/pull/196)
516
+ - Fixed formatting problems and typographical errors [#194](https://github.com/logstash-plugins/logstash-output-s3/pull/194), [#201](https://github.com/logstash-plugins/logstash-output-s3/pull/201), and [#204](https://github.com/logstash-plugins/logstash-output-s3/pull/204)
517
+
518
+ ## 4.1.8
519
+ - Add support for setting mutipart upload threshold [#202](https://github.com/logstash-plugins/logstash-output-s3/pull/202)
520
+
521
+ ## 4.1.7
522
+ - Fixed issue where on restart, 0 byte files could erroneously be uploaded to s3 [#195](https://github.com/logstash-plugins/logstash-output-s3/issues/195)
523
+
524
+ ## 4.1.6
525
+ - Fixed leak of file handles that prevented temporary files from being cleaned up before pipeline restart [#190](https://github.com/logstash-plugins/logstash-output-s3/issues/190)
526
+
527
+ ## 4.1.5
528
+ - Fixed bucket validation failures when bucket policy requires encryption [#191](https://github.com/logstash-plugins/logstash-output-s3/pull/191)
529
+
530
+ ## 4.1.4
531
+ - [#185](https://github.com/logstash-plugins/logstash-output-s3/pull/184) Internal: Revert rake pinning to fix upstream builds
532
+
533
+ ## 4.1.3
534
+ - [#181](https://github.com/logstash-plugins/logstash-output-s3/pull/181) Docs: Fix incorrect characterization of parameters as `required` in example configuration.
535
+ - [#184](https://github.com/logstash-plugins/logstash-output-s3/pull/184) Internal: Pin rake version for jruby-1.7 compatibility
536
+
537
+ ## 4.1.2
538
+ - Symbolize hash keys for additional_settings hash #179
539
+
540
+ ## 4.1.1
541
+ - Docs: Set the default_codec doc attribute.
542
+
543
+ ## 4.1.0
544
+ - Add documentation for endpoint, role_arn and role_session_name #174
545
+ - Add option for additional settings #173
546
+ - Add more S3 bucket ACLs #158
547
+ - Handle file not found exception on S3 upload #144
548
+ - Document prefix interpolation #154
549
+
550
+ ## 4.0.13
551
+ - Update gemspec summary
552
+
553
+ ## 4.0.12
554
+ - Fix bug where output would fail if the s3 bucket had encryption enabled (#146, #155)
555
+
556
+ ## 4.0.11
557
+ - Fixed a randomly occurring error that logged as a missing `__jcreate_meta` method
558
+
559
+ ## 4.0.10
560
+ - Fix some documentation issues
561
+
562
+ ## 4.0.9
563
+ - Correct issue that allows to run on Ruby 9k. #150
564
+
565
+ ## 4.0.8
566
+ - Documentation changes
567
+
568
+ ## 4.0.7
569
+ - Fix: `#restore_from_crash` should use the same upload options as the normal uploader. #140
570
+ - Fix: Wrongly named `canned_acl` options, renamed to "public-read", "public-read-write", "authenticated-read", from the documentation http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
571
+
572
+ ## 4.0.6
573
+ - Fix: Use the right `signature_version` for the SDK v2 #129
574
+ - Fix an issue to prevent the output to upload empty file to S3 #128
575
+ - Docs: Update the doc to show the new format of the remote file #126
576
+
577
+ ## 4.0.5
578
+ - Delete the file on disk after they are succesfully uploaded to S3 #122 #120
579
+ - Added logging when an exception occur in the Uploader's `on_complete` callback
580
+
581
+ ## 4.0.4
582
+ - Add support for `storage_class` configuration
583
+ - Fix compatibility with Logstash 2.4
584
+ - Add support for `aws:kms` server side encryption #104
585
+
586
+ ## 4.0.3
587
+ - When configuring the `canned_acl` options of the plugins the role was not applied correctly to the created object: #7
588
+
589
+ ## 4.0.2
590
+ - Fixed AWS authentication when using instance profile credentials.
591
+
592
+ ## 4.0.1
593
+ - Improved Error logging for S3 validation. Now specific S3 perms errors are logged
594
+
595
+ ## 4.0.0
596
+ - This version is a complete rewrite over version 3.0.0 See #103
597
+ - This Plugin now uses the V2 version of the SDK, this make sure we receive the latest updates and changes.
598
+ - We now uses S3's `upload_file` instead of reading chunks, this method is more efficient and will uses the multipart with threads if the files is too big.
599
+ - You can now use the `fieldref` syntax in the prefix to dynamically changes the target with the events it receives.
600
+ - The Upload queue is now a bounded list, this options is necessary to allow back pressure to be communicated back to the pipeline but its configurable by the user.
601
+ - If the queue is full the plugin will start the upload in the current thread.
602
+ - The plugin now threadsafe and support the concurrency model `shared`
603
+ - The rotation strategy can be selected, the recommended is `size_and_time` that will check for both the configured limits (`size` and `time` are also available)
604
+ - The `restore` option will now use a separate threadpool with an unbounded queue
605
+ - The `restore` option will not block the launch of logstash and will uses less resources than the real time path
606
+ - The plugin now uses `multi_receive_encode`, this will optimize the writes to the files
607
+ - rotate operation are now batched to reduce the number of IO calls.
608
+ - Empty file will not be uploaded by any rotation rotation strategy
609
+ - We now use Concurrent-Ruby for the implementation of the java executor
610
+ - If you have finer grain permission on prefixes or want faster boot, you can disable the credentials check with `validate_credentials_on_root_bucket`
611
+ - The credentials check will no longer fails if we can't delete the file
612
+ - We now have a full suite of integration test for all the defined rotation
613
+
614
+ Fixes: #4 #81 #44 #59 #50
615
+
616
+ ## 3.2.0
617
+ - Move to the new concurrency model `:single`
618
+ - use correct license identifier #99
619
+ - add support for `bucket_owner_full_control` in the canned ACL #87
620
+ - delete the test file but ignore any errors, because we actually only need to be able to write to S3. #97
621
+
622
+ ## 3.1.2
623
+ - Fix improper shutdown of output worker threads
624
+ - improve exception handling
625
+
626
+ ## 3.0.1
627
+ - Republish all the gems under jruby.
628
+
629
+ ## 3.0.0
630
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
631
+
632
+ ## 2.0.7
633
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
634
+
635
+ ## 2.0.6
636
+ - New dependency requirements for logstash-core for the 5.0 release
637
+
638
+ ## 2.0.5
639
+ - Support signature_version option for v4 S3 keys
640
+
641
+ ## 2.0.4
642
+ - Remove the `Time.now` stub in the spec, it was conflicting with other test when running inside the default plugins test #63
643
+ - Make the spec run faster by adjusting the values of time rotation test.
644
+
645
+ ## 2.0.3
646
+ - Update deps for logstash 2.0
647
+
648
+ ## 2.0.2
649
+ - Fixes an issue when tags were defined #39
650
+
651
+ ## 2.0.0
652
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
653
+ instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
654
+ - Dependency on logstash-core update to 2.0
655
+
656
+ ## 1.0.1
657
+ - Fix a synchronization issue when doing file rotation and checking the size of the current file
658
+ - Fix an issue with synchronization when shutting down the plugin and closing the current temp file
data/CHANGELOG.md ADDED
@@ -0,0 +1,15 @@
1
+ ## 1.0.0
2
+
3
+ * Added the initial set of Logstash AWS plugins that ship with Logstash.
4
+ You can find the merged changelog of the individual plugins in CHANGELOG.PRE.MERGE.md.
5
+ These are (along with the version that was used for import):
6
+ - logstash-codec-cloudfront (3.0.3)
7
+ - logstash-codec-cloudtrail (3.0.5)
8
+ - logstash-input-cloudwatch (2.2.4)
9
+ - logstash-input-s3 (3.8.4)
10
+ - logstash-input-sqs (3.3.2)
11
+ - logstash-mixin-aws (5.1.0)
12
+ - logstash-output-cloudwatch (3.0.10)
13
+ - logstash-output-sns (4.0.8)
14
+ - logstash-output-sqs (6.0.1)
15
+ - logstash-output-s3 (4.3.5)