logstash-integration-aws 0.1.0.pre

Sign up to get free protection for your applications and to get access to all the features.
Files changed (89) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.PRE.MERGE.md +658 -0
  3. data/CHANGELOG.md +15 -0
  4. data/CONTRIBUTORS +40 -0
  5. data/Gemfile +11 -0
  6. data/LICENSE +202 -0
  7. data/NOTICE.TXT +5 -0
  8. data/README.md +205 -0
  9. data/docs/codec-cloudfront.asciidoc +53 -0
  10. data/docs/codec-cloudtrail.asciidoc +45 -0
  11. data/docs/index.asciidoc +38 -0
  12. data/docs/input-cloudwatch.asciidoc +320 -0
  13. data/docs/input-s3.asciidoc +346 -0
  14. data/docs/input-sqs.asciidoc +287 -0
  15. data/docs/output-cloudwatch.asciidoc +321 -0
  16. data/docs/output-s3.asciidoc +442 -0
  17. data/docs/output-sns.asciidoc +166 -0
  18. data/docs/output-sqs.asciidoc +242 -0
  19. data/lib/logstash/codecs/cloudfront.rb +84 -0
  20. data/lib/logstash/codecs/cloudtrail.rb +47 -0
  21. data/lib/logstash/inputs/cloudwatch.rb +338 -0
  22. data/lib/logstash/inputs/s3.rb +466 -0
  23. data/lib/logstash/inputs/sqs.rb +196 -0
  24. data/lib/logstash/outputs/cloudwatch.rb +346 -0
  25. data/lib/logstash/outputs/s3/file_repository.rb +121 -0
  26. data/lib/logstash/outputs/s3/path_validator.rb +18 -0
  27. data/lib/logstash/outputs/s3/size_and_time_rotation_policy.rb +24 -0
  28. data/lib/logstash/outputs/s3/size_rotation_policy.rb +26 -0
  29. data/lib/logstash/outputs/s3/temporary_file.rb +71 -0
  30. data/lib/logstash/outputs/s3/temporary_file_factory.rb +129 -0
  31. data/lib/logstash/outputs/s3/time_rotation_policy.rb +26 -0
  32. data/lib/logstash/outputs/s3/uploader.rb +74 -0
  33. data/lib/logstash/outputs/s3/writable_directory_validator.rb +17 -0
  34. data/lib/logstash/outputs/s3/write_bucket_permission_validator.rb +60 -0
  35. data/lib/logstash/outputs/s3.rb +405 -0
  36. data/lib/logstash/outputs/sns.rb +133 -0
  37. data/lib/logstash/outputs/sqs.rb +167 -0
  38. data/lib/logstash/plugin_mixins/aws_config/generic.rb +54 -0
  39. data/lib/logstash/plugin_mixins/aws_config/v2.rb +93 -0
  40. data/lib/logstash/plugin_mixins/aws_config.rb +8 -0
  41. data/logstash-integration-aws.gemspec +52 -0
  42. data/spec/codecs/cloudfront_spec.rb +92 -0
  43. data/spec/codecs/cloudtrail_spec.rb +56 -0
  44. data/spec/fixtures/aws_credentials_file_sample_test.yml +2 -0
  45. data/spec/fixtures/aws_temporary_credentials_file_sample_test.yml +3 -0
  46. data/spec/fixtures/cloudfront.log +4 -0
  47. data/spec/fixtures/compressed.log.gee.zip +0 -0
  48. data/spec/fixtures/compressed.log.gz +0 -0
  49. data/spec/fixtures/compressed.log.gzip +0 -0
  50. data/spec/fixtures/invalid_utf8.gbk.log +2 -0
  51. data/spec/fixtures/json.log +2 -0
  52. data/spec/fixtures/json_with_message.log +2 -0
  53. data/spec/fixtures/multiline.log +6 -0
  54. data/spec/fixtures/multiple_compressed_streams.gz +0 -0
  55. data/spec/fixtures/uncompressed.log +2 -0
  56. data/spec/inputs/cloudwatch_spec.rb +85 -0
  57. data/spec/inputs/s3_spec.rb +610 -0
  58. data/spec/inputs/sincedb_spec.rb +17 -0
  59. data/spec/inputs/sqs_spec.rb +324 -0
  60. data/spec/integration/cloudwatch_spec.rb +25 -0
  61. data/spec/integration/dynamic_prefix_spec.rb +92 -0
  62. data/spec/integration/gzip_file_spec.rb +62 -0
  63. data/spec/integration/gzip_size_rotation_spec.rb +63 -0
  64. data/spec/integration/outputs/sqs_spec.rb +98 -0
  65. data/spec/integration/restore_from_crash_spec.rb +67 -0
  66. data/spec/integration/s3_spec.rb +66 -0
  67. data/spec/integration/size_rotation_spec.rb +59 -0
  68. data/spec/integration/sqs_spec.rb +110 -0
  69. data/spec/integration/stress_test_spec.rb +60 -0
  70. data/spec/integration/time_based_rotation_with_constant_write_spec.rb +60 -0
  71. data/spec/integration/time_based_rotation_with_stale_write_spec.rb +64 -0
  72. data/spec/integration/upload_current_file_on_shutdown_spec.rb +51 -0
  73. data/spec/outputs/cloudwatch_spec.rb +38 -0
  74. data/spec/outputs/s3/file_repository_spec.rb +143 -0
  75. data/spec/outputs/s3/size_and_time_rotation_policy_spec.rb +77 -0
  76. data/spec/outputs/s3/size_rotation_policy_spec.rb +41 -0
  77. data/spec/outputs/s3/temporary_file_factory_spec.rb +89 -0
  78. data/spec/outputs/s3/temporary_file_spec.rb +47 -0
  79. data/spec/outputs/s3/time_rotation_policy_spec.rb +60 -0
  80. data/spec/outputs/s3/uploader_spec.rb +69 -0
  81. data/spec/outputs/s3/writable_directory_validator_spec.rb +40 -0
  82. data/spec/outputs/s3/write_bucket_permission_validator_spec.rb +49 -0
  83. data/spec/outputs/s3_spec.rb +232 -0
  84. data/spec/outputs/sns_spec.rb +160 -0
  85. data/spec/plugin_mixin/aws_config_spec.rb +217 -0
  86. data/spec/spec_helper.rb +8 -0
  87. data/spec/support/helpers.rb +119 -0
  88. data/spec/unit/outputs/sqs_spec.rb +247 -0
  89. metadata +467 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 9aed616c6b3abd22f2196b4939ffbedbf1d2c8785a5aef119fc9fe8242847921
4
+ data.tar.gz: e80bfc1603c49e9271111c8ff4767e42ddcfaa8dedbca2100e3b9e76ae10d735
5
+ SHA512:
6
+ metadata.gz: 5a9c24ffdc56ec13be5b32e27016fc55d4a4a256b16f6d8f28fa38128b1cb1433abd19e63b245ab6d693539af612ba29ad0f345c5a28e96c64a52d66f867c778
7
+ data.tar.gz: b14a38aea9eafb67c3b0d516463c43540f807f5f3b63fe459841023becd7003ebdb188f202dc7b3233c40b995c3a13ca41a1bcdf31e55abc124eac0954ff0da9
@@ -0,0 +1,658 @@
1
+ # CHANGELOG - logstash-codec-cloudfront
2
+
3
+ ## 3.0.3
4
+ - Update gemspec summary
5
+
6
+ ## 3.0.2
7
+ - Fix some documentation issues
8
+
9
+ # 2.0.4
10
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
11
+ # 2.0.3
12
+ - New dependency requirements for logstash-core for the 5.0 release
13
+ ## 2.0.0
14
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
15
+ instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
16
+ - Dependency on logstash-core update to 2.0
17
+
18
+ ---------
19
+
20
+ # CHANGELOG - logstash-codec-cloudtrail
21
+
22
+ ## 3.0.5
23
+ - [#22](https://github.com/logstash-plugins/logstash-codec-cloudtrail/pull/22)Handle 'sourceIpAddress' fields with non-ip address content by moving them to 'sourceHost' field
24
+
25
+ ## 3.0.4
26
+ - Don't crash when data doesn't contain some particular elements
27
+
28
+ ## 3.0.3
29
+ - Fix some documentation issues
30
+
31
+ # 3.0.1
32
+ - fixed mapping template for requestParameters.disableApiTermination field
33
+
34
+ ## 3.0.0
35
+ - Update to support Logstash 2.4 & 5.0 APIs
36
+
37
+ ## 2.0.4
38
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
39
+
40
+ ## 2.0.3
41
+ - New dependency requirements for logstash-core for the 5.0 release
42
+
43
+ ## 2.0.0
44
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
45
+ instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
46
+ - Dependency on logstash-core update to 2.0
47
+
48
+ ---------
49
+
50
+ # CHANGELOG - logstash-input-cloudwatch
51
+
52
+ ## 2.2.4
53
+ - Fixed shutdown handling [#43](https://github.com/logstash-plugins/logstash-input-cloudwatch/pull/43)
54
+
55
+ ## 2.2.3
56
+ - Fixed issue where metric timestamp was being lost due to over-writing by end_time [#38](https://github.com/logstash-plugins/logstash-input-cloudwatch/pull/38)
57
+
58
+ ## 2.2.2
59
+ - Added ability to use AWS/EC2 namespace without requiring filters [#36](https://github.com/logstash-plugins/logstash-input-cloudwatch/pull/36)
60
+
61
+ ## 2.2.1
62
+ - Fixed README.md link to request metric support to point to this repo [#34](https://github.com/logstash-plugins/logstash-input-cloudwatch/pull/34)
63
+
64
+ ## 2.2.0
65
+ - Changed to use the underlying version of the AWS SDK to v2. [#32](https://github.com/logstash-plugins/logstash-input-cloudwatch/pull/32)
66
+ - Fixed License definition in gemspec to be valid SPDX identifier [#32](https://github.com/logstash-plugins/logstash-input-cloudwatch/pull/32)
67
+ - Fixed fatal error when using secret key attribute in config [#30](https://github.com/logstash-plugins/logstash-input-cloudwatch/issues/30)
68
+
69
+ ## 2.1.1
70
+ - Docs: Set the default_codec doc attribute.
71
+
72
+ ## 2.1.0
73
+ - Add documentation for endpoint, role_arn and role_session_name #29
74
+ - Reduce info level logging verbosity #27
75
+
76
+ ## 2.0.3
77
+ - Update gemspec summary
78
+
79
+ ## 2.0.2
80
+ - Fix some documentation issues
81
+
82
+ # 1.1.3
83
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
84
+ # 1.1.1
85
+ - New dependency requirements for logstash-core for the 5.0 release
86
+ ## 1.1.0
87
+ - Moved from jrgns/logstash-input-cloudwatch to logstash-plugins
88
+
89
+ ---------
90
+
91
+ # CHANGELOG - logstash-input-s3
92
+
93
+ ## 3.8.4
94
+ - Refactoring, reuse code to manage `additional_settings` from mixin-aws [#237](https://github.com/logstash-plugins/logstash-input-s3/pull/237)
95
+
96
+ ## 3.8.3
97
+ - Fix missing `metadata` and `type` of the last event [#223](https://github.com/logstash-plugins/logstash-input-s3/pull/223)
98
+
99
+ ## 3.8.2
100
+ - Refactor: read sincedb time once per bucket listing [#233](https://github.com/logstash-plugins/logstash-input-s3/pull/233)
101
+
102
+ ## 3.8.1
103
+ - Feat: cast true/false values for additional_settings [#232](https://github.com/logstash-plugins/logstash-input-s3/pull/232)
104
+
105
+ ## 3.8.0
106
+ - Add ECS v8 support.
107
+
108
+ ## 3.7.0
109
+ - Add ECS support. [#228](https://github.com/logstash-plugins/logstash-input-s3/pull/228)
110
+ - Fix missing file in cutoff time change. [#224](https://github.com/logstash-plugins/logstash-input-s3/pull/224)
111
+
112
+ ## 3.6.0
113
+ - Fixed unprocessed file with the same `last_modified` in ingestion. [#220](https://github.com/logstash-plugins/logstash-input-s3/pull/220)
114
+
115
+ ## 3.5.2
116
+ - [DOC]Added note that only AWS S3 is supported. No other S3 compatible storage solutions are supported. [#208](https://github.com/logstash-plugins/logstash-input-s3/issues/208)
117
+
118
+ ## 3.5.1
119
+ - [DOC]Added example for `exclude_pattern` and reordered option descriptions [#204](https://github.com/logstash-plugins/logstash-input-s3/issues/204)
120
+
121
+ ## 3.5.0
122
+ - Added support for including objects restored from Glacier or Glacier Deep [#199](https://github.com/logstash-plugins/logstash-input-s3/issues/199)
123
+ - Added `gzip_pattern` option, enabling more flexible determination of whether a file is gzipped [#165](https://github.com/logstash-plugins/logstash-input-s3/issues/165)
124
+ - Refactor: log exception: class + unify logging messages a bit [#201](https://github.com/logstash-plugins/logstash-input-s3/pull/201)
125
+
126
+ ## 3.4.1
127
+ - Fixed link formatting for input type (documentation)
128
+
129
+ ## 3.4.0
130
+ - Skips objects that are archived to AWS Glacier with a helpful log message (previously they would log as matched, but then fail to load events) [#160](https://github.com/logstash-plugins/logstash-input-s3/pull/160)
131
+ - Added `watch_for_new_files` option, enabling single-batch imports [#159](https://github.com/logstash-plugins/logstash-input-s3/pull/159)
132
+
133
+ ## 3.3.7
134
+ - Added ability to optionally include S3 object properties inside @metadata [#155](https://github.com/logstash-plugins/logstash-input-s3/pull/155)
135
+
136
+ ## 3.3.6
137
+ - Fixed error in documentation by removing illegal commas [#154](https://github.com/logstash-plugins/logstash-input-s3/pull/154)
138
+
139
+ ## 3.3.5
140
+ - [#136](https://github.com/logstash-plugins/logstash-input-s3/pull/136) Avoid plugin crashes when encountering 'bad' files in S3 buckets
141
+
142
+ ## 3.3.4
143
+ - Log entry when bucket is empty #150
144
+
145
+ ## 3.3.3
146
+ - Symbolize hash keys for additional_settings hash #148
147
+
148
+ ## 3.3.2
149
+ - Docs: Set the default_codec doc attribute.
150
+
151
+ ## 3.3.1
152
+ - Improve error handling when listing/downloading from S3 #144
153
+
154
+ ## 3.3.0
155
+ - Add documentation for endpoint, role_arn and role_session_name #142
156
+ - Add support for additional_settings option #141
157
+
158
+ ## 3.2.0
159
+ - Add support for auto-detecting gzip files with `.gzip` extension, in addition to existing support for `*.gz`
160
+ - Improve performance of gzip decoding by 10x by using Java's Zlib
161
+
162
+ ## 3.1.9
163
+ - Change default sincedb path to live in `{path.data}/plugins/inputs/s3` instead of $HOME.
164
+ Prior Logstash installations (using $HOME default) are automatically migrated.
165
+ - Don't download the file if the length is 0 #2
166
+
167
+ ## 3.1.8
168
+ - Update gemspec summary
169
+
170
+ ## 3.1.7
171
+ - Fix missing last multi-line entry #120
172
+
173
+ ## 3.1.6
174
+ - Fix some documentation issues
175
+
176
+ ## 3.1.4
177
+ - Avoid parsing non string elements #109
178
+
179
+ ## 3.1.3
180
+ - The plugin will now include the s3 key in the metadata #105
181
+
182
+ ## 3.1.2
183
+ - Fix an issue when the remote file contains multiple blob of gz in the same file #101
184
+ - Make the integration suite run
185
+ - Remove uneeded development dependency
186
+
187
+ ## 3.1.1
188
+ - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
189
+
190
+ ## 3.1.0
191
+ - breaking,config: Remove deprecated config `credentials` and `region_endpoint`. Please use AWS mixin.
192
+
193
+ ## 3.0.1
194
+ - Republish all the gems under jruby.
195
+
196
+ ## 3.0.0
197
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
198
+
199
+ ## 2.0.6
200
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
201
+
202
+ ## 2.0.5
203
+ - New dependency requirements for logstash-core for the 5.0 release
204
+
205
+ ## 2.0.4
206
+ - Fix for Error: No Such Key problem when deleting
207
+
208
+ ## 2.0.3
209
+ - Do not raise an exception if the sincedb file is empty, instead return the current time #66
210
+
211
+ ## 2.0.0
212
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
213
+ instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
214
+ - Dependency on logstash-core update to 2.0
215
+
216
+ ---------
217
+
218
+ # CHANGELOG - logstash-input-sqs
219
+
220
+ ## 3.3.2
221
+ - Fix an issue that prevented timely shutdown when subscribed to an inactive queue [#65](https://github.com/logstash-plugins/logstash-input-sqs/pull/65)
222
+
223
+ ## 3.3.1
224
+ - Refactoring: used logstash-mixin-aws to leverage shared code to manage `additional_settings` [#64](https://github.com/logstash-plugins/logstash-input-sqs/pull/64)
225
+
226
+ ## 3.3.0
227
+ - Feature: Add `additional_settings` option to fine-grain configuration of AWS client [#61](https://github.com/logstash-plugins/logstash-input-sqs/pull/61)
228
+
229
+ ## 3.2.0
230
+ - Feature: Add `queue_owner_aws_account_id` parameter for cross-account queues [#60](https://github.com/logstash-plugins/logstash-input-sqs/pull/60)
231
+
232
+ ## 3.1.3
233
+ - Fix: retry networking errors (with backoff) [#57](https://github.com/logstash-plugins/logstash-input-sqs/pull/57)
234
+
235
+ ## 3.1.2
236
+ - Added support for multiple events inside same message from SQS [#48](https://github.com/logstash-plugins/logstash-input-sqs/pull/48/files)
237
+
238
+ ## 3.1.1
239
+ - Docs: Set the default_codec doc attribute.
240
+
241
+ ## 3.1.0
242
+ - Add documentation for endpoint, role_arn and role_session_name #46
243
+ - Fix sample IAM policy to match to match the documentation #32
244
+
245
+ ## 3.0.6
246
+ - Update gemspec summary
247
+
248
+ ## 3.0.5
249
+ - Fix some documentation issues
250
+
251
+ ## 3.0.3
252
+ - Monkey-patch the AWS-SDK to prevent "uninitialized constant" errors.
253
+
254
+ ## 3.0.2
255
+ - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
256
+
257
+ ## 3.0.1
258
+ - Republish all the gems under jruby.
259
+ ## 3.0.0
260
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
261
+ # 2.0.5
262
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
263
+ # 2.0.4
264
+ - New dependency requirements for logstash-core for the 5.0 release
265
+ ## 2.0.3
266
+ - Fixes #22, wrong key use on the stats object
267
+ ## 2.0.0
268
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
269
+ instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
270
+ - Dependency on logstash-core update to 2.0
271
+
272
+ # 1.1.0
273
+ - AWS ruby SDK v2 upgrade
274
+ - Replaces aws-sdk dependencies with mixin-aws
275
+ - Removes unnecessary de-allocation
276
+ - Move the code into smaller methods to allow easier mocking and testing
277
+ - Add the option to configure polling frequency
278
+ - Adding a monkey patch to make sure `LogStash::ShutdownSignal` doesn't get catch by AWS RetryError.
279
+
280
+ ---------
281
+
282
+ # CHANGELOG - logstash-mixin-aws
283
+
284
+ ## 5.1.0
285
+ - Add support for 'addition_settings' configuration options used by S3 and SQS input plugins [#53](https://github.com/logstash-plugins/logstash-mixin-aws/pull/53).
286
+
287
+ ## 5.0.0
288
+ - Drop support for aws-sdk-v1
289
+
290
+ ## 4.4.1
291
+ - Fix: proxy with assumed role (properly) [#50](https://github.com/logstash-plugins/logstash-mixin-aws/pull/50).
292
+
293
+ ## 4.4.0
294
+ - Fix: credentials/proxy with assumed role [#48](https://github.com/logstash-plugins/logstash-mixin-aws/pull/48).
295
+ Plugin no longer assumes `access_key_id`/`secret_access_key` credentials not to be set when `role_arn` specified.
296
+
297
+ ## 4.3.0
298
+ - Drop strict value validation for region option #36
299
+ - Add endpoint option to customize the endpoint uri #32
300
+ - Allow user to provide a role to assume #27
301
+ - Update aws-sdk dependency to '~> 2'
302
+
303
+ ## 4.2.4
304
+ - Minor config validation fixes
305
+
306
+ ## 4.2.3
307
+ - Fix some documentation issues
308
+
309
+ ## 4.2.1
310
+ - Add eu-west-2, us-east-2 and ca-central-1 regions
311
+
312
+ ## 4.2.0
313
+ - Add region ap-south-1
314
+
315
+ ## 4.1.0
316
+ - Update aws-sdk to ~> 2.3.0
317
+
318
+ ## 4.0.2
319
+ - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
320
+
321
+ ## 4.0.1
322
+ - Republish all the gems under jruby.
323
+ ## 4.0.0
324
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
325
+ # 2.0.4
326
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
327
+ # 2.0.3
328
+ - New dependency requirements for logstash-core for the 5.0 release
329
+ ## 2.0.0
330
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
331
+ instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
332
+ - Dependency on logstash-core update to 2.0
333
+
334
+ # 1.0.1
335
+ * Correctly set proxy options on V2 of the aws-sdk
336
+
337
+ # 1.0.0
338
+ * Allow to use either V1 or V2 of the `AWS-SDK` in your plugins. Fixes: https://github.com/logstash-plugins/logstash-mixin-aws/issues/8
339
+
340
+ ---------
341
+
342
+ # CHANGELOG - logstash-output-cloudwatch
343
+
344
+ ## 3.0.10
345
+ - Deps: unpin rufus scheduler [#20](https://github.com/logstash-plugins/logstash-output-cloudwatch/pull/20)
346
+ - Fix: an old undefined method error which would surface with load (as queue fills up)
347
+
348
+ ## 3.0.9
349
+ - Fix: dropped usage of SHUTDOWN event deprecated since Logstash 5.0 [#18](https://github.com/logstash-plugins/logstash-output-cloudwatch/pull/18)
350
+
351
+ ## 3.0.8
352
+ - Docs: Set the default_codec doc attribute.
353
+
354
+ ## 3.0.7
355
+ - Update gemspec summary
356
+
357
+ ## 3.0.6
358
+ - Fix some documentation issues
359
+
360
+ ## 3.0.4
361
+ - Fix some remaining uses of the old event api. blocking the use of this plugin
362
+
363
+ ## 3.0.3
364
+ - Move some log messages from info to debug to avoid noise
365
+
366
+ ## 3.0.2
367
+ - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
368
+
369
+ ## 3.0.1
370
+ - Republish all the gems under jruby.
371
+ ## 3.0.0
372
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
373
+ # 2.0.4
374
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
375
+ # 2.0.3
376
+ - New dependency requirements for logstash-core for the 5.0 release
377
+ ## 2.0.0
378
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
379
+ instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
380
+ - Dependency on logstash-core update to 2.0
381
+
382
+ ---------
383
+
384
+ # CHANGELOG - logstash-output-sns
385
+
386
+ ## 4.0.8
387
+ - Feat: handle host object as subject (due ECS) [#22](https://github.com/logstash-plugins/logstash-output-sns/pull/22)
388
+
389
+ ## 4.0.7
390
+ - Docs: Set the default_codec doc attribute.
391
+
392
+ ## 4.0.6
393
+ - Update gemspec summary
394
+
395
+ ## 4.0.5
396
+ - Fix some documentation issues
397
+
398
+ ## 4.0.3
399
+ - Mark this output as thread safe to allow concurrent connections to AWS.
400
+
401
+ ## 4.0.2
402
+ - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
403
+
404
+ ## 4.0.1
405
+ - Republish all the gems under jruby.
406
+ ## 4.0.0
407
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
408
+ # 3.0.4
409
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
410
+ # 3.0.3
411
+ - New dependency requirements for logstash-core for the 5.0 release
412
+ ## 3.0.0
413
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
414
+ instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
415
+ - Dependency on logstash-core update to 2.0
416
+
417
+ # 1.0.1
418
+ * Properly trim messages for AWS without breaking unicode byte boundaries
419
+
420
+ # 1.0.0
421
+ * Full refactor.
422
+ * This plugin now uses codecs for all formatting. The 'format' option has now been removed. Please use a codec.
423
+ # 0.1.5
424
+ * If no `subject` are specified fallback to the %{host} key (https://github.com/logstash-plugins/logstash-output-sns/pull/2)
425
+ * Migrate the SNS Api to use the AWS-SDK v2
426
+
427
+ ---------
428
+
429
+ # CHANGELOG - logstash-output-sqs
430
+
431
+ ## 6.0.1
432
+ - Added missing index entry for `queue_owner_aws_account_id` [#33](https://github.com/logstash-plugins/logstash-output-sqs/pull/33)
433
+
434
+ ## 6.0.0
435
+ - Removed obsolete fields `batch` and `batch_timeout`
436
+ - Removed workaround to JRuby bug (see more [here](https://github.com/jruby/jruby/issues/3645))
437
+
438
+ ## 5.1.2
439
+ - Added the ability to send to a different account id's queue. [#30](https://github.com/logstash-plugins/logstash-output-sqs/pull/30)
440
+
441
+ ## 5.1.1
442
+ - Docs: Set the default_codec doc attribute.
443
+
444
+ ## 5.1.0
445
+ - Add documentation for endpoint, role_arn and role_session_name #29
446
+
447
+ ## 5.0.2
448
+ - Update gemspec summary
449
+
450
+ ## 5.0.1
451
+ - Fix some documentation issues
452
+
453
+ ## 5.0.0
454
+ - Breaking: mark deprecated `batch` and `batch_timeout` options as obsolete
455
+
456
+ ## 4.0.1
457
+ - Docs: Fix doc generation issue by removing extraneous comments.
458
+
459
+ ## 4.0.0
460
+ - Add unit and integration tests.
461
+ - Adjust the sample IAM policy in the documentation, removing actions which are not actually required by the plugin. Specifically, the following actions are not required: `sqs:ChangeMessageVisibility`, `sqs:ChangeMessageVisibilityBatch`, `sqs:GetQueueAttributes` and `sqs:ListQueues`.
462
+ - Dynamically adjust the batch message size. SQS allows up to 10 messages to be published in a single batch, however the total size of the batch is limited to 256KiB (see [Limits in Amazon SQS](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/limits-messages.html)). This plugin will now dynamically adjust the number of events included in each batch to ensure that the total batch size does not exceed `message_max_size`. Note that any single messages which exceeds the 256KiB size limit will be dropped.
463
+ - Move to the new concurrency model, `:shared`.
464
+ - The `batch_timeout` parameter has been deprecated because it no longer has any effect.
465
+ - The individual (non-batch) mode of operation (i.e. `batch => false`) has been deprecated. Batch mode is vastly more performant and we do not believe that there are any use cases which require non-batch mode. You can emulate non-batch mode by setting `batch_events => 1`, although this will call `sqs:SendMessageBatch` with a batch size of 1 rather than calling `sqs:SendMessage`.
466
+ - The plugin now implements `#multi_receive_encoded` and no longer uses `Stud::Buffer`.
467
+ - Update the AWS SDK to version 2.
468
+
469
+ ## 3.0.2
470
+ - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
471
+
472
+ ## 3.0.1
473
+ - Republish all the gems under jruby.
474
+ ## 3.0.0
475
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
476
+ # 2.0.5
477
+ - Add travis config and build status
478
+ - Require the AWS mixin to be higher than 1.0.0
479
+ # 2.0.4
480
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
481
+ # 2.0.3
482
+ - New dependency requirements for logstash-core for the 5.0 release
483
+ ## 2.0.0
484
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
485
+ instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
486
+ - Dependency on logstash-core update to 2.0
487
+
488
+ # CHANGELOG - logstash - output-s3
489
+
490
+ ## 4.3.5
491
+ - Feat: cast true/false values for additional_settings [#241](https://github.com/logstash-plugins/logstash-output-s3/pull/241)
492
+
493
+ ## 4.3.4
494
+ - [DOC] Added note about performance implications of interpolated strings in prefixes [#233](https://github.com/logstash-plugins/logstash-output-s3/pull/233)
495
+
496
+ ## 4.3.3
497
+ - [DOC] Updated links to use shared attributes [#230](https://github.com/logstash-plugins/logstash-output-s3/pull/230)
498
+
499
+ ## 4.3.2
500
+ - [DOC] Added note that only AWS S3 is supported. No other S3 compatible storage solutions are supported. [#223](https://github.com/logstash-plugins/logstash-output-s3/pull/223)
501
+
502
+ ## 4.3.1
503
+ - [DOC] Updated setting descriptions for clarity [#219](https://github.com/logstash-plugins/logstash-output-s3/pull/219) and [#220](https://github.com/logstash-plugins/logstash-output-s3/pull/220)
504
+
505
+ ## 4.3.0
506
+ - Feat: Added retry_count and retry_delay config [#218](https://github.com/logstash-plugins/logstash-output-s3/pull/218)
507
+
508
+ ## 4.2.0
509
+ - Added ability to specify [ONEZONE_IA](https://aws.amazon.com/s3/storage-classes/#__) as storage_class
510
+
511
+ ## 4.1.10
512
+ - Added clarification for endpoint in documentation [#198](https://github.com/logstash-plugins/logstash-output-s3/pull/198)
513
+
514
+ ## 4.1.9
515
+ - Added configuration information for multiple s3 outputs to documentation [#196](https://github.com/logstash-plugins/logstash-output-s3/pull/196)
516
+ - Fixed formatting problems and typographical errors [#194](https://github.com/logstash-plugins/logstash-output-s3/pull/194), [#201](https://github.com/logstash-plugins/logstash-output-s3/pull/201), and [#204](https://github.com/logstash-plugins/logstash-output-s3/pull/204)
517
+
518
+ ## 4.1.8
519
+ - Add support for setting mutipart upload threshold [#202](https://github.com/logstash-plugins/logstash-output-s3/pull/202)
520
+
521
+ ## 4.1.7
522
+ - Fixed issue where on restart, 0 byte files could erroneously be uploaded to s3 [#195](https://github.com/logstash-plugins/logstash-output-s3/issues/195)
523
+
524
+ ## 4.1.6
525
+ - Fixed leak of file handles that prevented temporary files from being cleaned up before pipeline restart [#190](https://github.com/logstash-plugins/logstash-output-s3/issues/190)
526
+
527
+ ## 4.1.5
528
+ - Fixed bucket validation failures when bucket policy requires encryption [#191](https://github.com/logstash-plugins/logstash-output-s3/pull/191)
529
+
530
+ ## 4.1.4
531
+ - [#185](https://github.com/logstash-plugins/logstash-output-s3/pull/184) Internal: Revert rake pinning to fix upstream builds
532
+
533
+ ## 4.1.3
534
+ - [#181](https://github.com/logstash-plugins/logstash-output-s3/pull/181) Docs: Fix incorrect characterization of parameters as `required` in example configuration.
535
+ - [#184](https://github.com/logstash-plugins/logstash-output-s3/pull/184) Internal: Pin rake version for jruby-1.7 compatibility
536
+
537
+ ## 4.1.2
538
+ - Symbolize hash keys for additional_settings hash #179
539
+
540
+ ## 4.1.1
541
+ - Docs: Set the default_codec doc attribute.
542
+
543
+ ## 4.1.0
544
+ - Add documentation for endpoint, role_arn and role_session_name #174
545
+ - Add option for additional settings #173
546
+ - Add more S3 bucket ACLs #158
547
+ - Handle file not found exception on S3 upload #144
548
+ - Document prefix interpolation #154
549
+
550
+ ## 4.0.13
551
+ - Update gemspec summary
552
+
553
+ ## 4.0.12
554
+ - Fix bug where output would fail if the s3 bucket had encryption enabled (#146, #155)
555
+
556
+ ## 4.0.11
557
+ - Fixed a randomly occurring error that logged as a missing `__jcreate_meta` method
558
+
559
+ ## 4.0.10
560
+ - Fix some documentation issues
561
+
562
+ ## 4.0.9
563
+ - Correct issue that allows to run on Ruby 9k. #150
564
+
565
+ ## 4.0.8
566
+ - Documentation changes
567
+
568
+ ## 4.0.7
569
+ - Fix: `#restore_from_crash` should use the same upload options as the normal uploader. #140
570
+ - Fix: Wrongly named `canned_acl` options, renamed to "public-read", "public-read-write", "authenticated-read", from the documentation http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
571
+
572
+ ## 4.0.6
573
+ - Fix: Use the right `signature_version` for the SDK v2 #129
574
+ - Fix an issue to prevent the output to upload empty file to S3 #128
575
+ - Docs: Update the doc to show the new format of the remote file #126
576
+
577
+ ## 4.0.5
578
+ - Delete the file on disk after they are succesfully uploaded to S3 #122 #120
579
+ - Added logging when an exception occur in the Uploader's `on_complete` callback
580
+
581
+ ## 4.0.4
582
+ - Add support for `storage_class` configuration
583
+ - Fix compatibility with Logstash 2.4
584
+ - Add support for `aws:kms` server side encryption #104
585
+
586
+ ## 4.0.3
587
+ - When configuring the `canned_acl` options of the plugins the role was not applied correctly to the created object: #7
588
+
589
+ ## 4.0.2
590
+ - Fixed AWS authentication when using instance profile credentials.
591
+
592
+ ## 4.0.1
593
+ - Improved Error logging for S3 validation. Now specific S3 perms errors are logged
594
+
595
+ ## 4.0.0
596
+ - This version is a complete rewrite over version 3.0.0 See #103
597
+ - This Plugin now uses the V2 version of the SDK, this make sure we receive the latest updates and changes.
598
+ - We now uses S3's `upload_file` instead of reading chunks, this method is more efficient and will uses the multipart with threads if the files is too big.
599
+ - You can now use the `fieldref` syntax in the prefix to dynamically changes the target with the events it receives.
600
+ - The Upload queue is now a bounded list, this options is necessary to allow back pressure to be communicated back to the pipeline but its configurable by the user.
601
+ - If the queue is full the plugin will start the upload in the current thread.
602
+ - The plugin now threadsafe and support the concurrency model `shared`
603
+ - The rotation strategy can be selected, the recommended is `size_and_time` that will check for both the configured limits (`size` and `time` are also available)
604
+ - The `restore` option will now use a separate threadpool with an unbounded queue
605
+ - The `restore` option will not block the launch of logstash and will uses less resources than the real time path
606
+ - The plugin now uses `multi_receive_encode`, this will optimize the writes to the files
607
+ - rotate operation are now batched to reduce the number of IO calls.
608
+ - Empty file will not be uploaded by any rotation rotation strategy
609
+ - We now use Concurrent-Ruby for the implementation of the java executor
610
+ - If you have finer grain permission on prefixes or want faster boot, you can disable the credentials check with `validate_credentials_on_root_bucket`
611
+ - The credentials check will no longer fails if we can't delete the file
612
+ - We now have a full suite of integration test for all the defined rotation
613
+
614
+ Fixes: #4 #81 #44 #59 #50
615
+
616
+ ## 3.2.0
617
+ - Move to the new concurrency model `:single`
618
+ - use correct license identifier #99
619
+ - add support for `bucket_owner_full_control` in the canned ACL #87
620
+ - delete the test file but ignore any errors, because we actually only need to be able to write to S3. #97
621
+
622
+ ## 3.1.2
623
+ - Fix improper shutdown of output worker threads
624
+ - improve exception handling
625
+
626
+ ## 3.0.1
627
+ - Republish all the gems under jruby.
628
+
629
+ ## 3.0.0
630
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
631
+
632
+ ## 2.0.7
633
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
634
+
635
+ ## 2.0.6
636
+ - New dependency requirements for logstash-core for the 5.0 release
637
+
638
+ ## 2.0.5
639
+ - Support signature_version option for v4 S3 keys
640
+
641
+ ## 2.0.4
642
+ - Remove the `Time.now` stub in the spec, it was conflicting with other test when running inside the default plugins test #63
643
+ - Make the spec run faster by adjusting the values of time rotation test.
644
+
645
+ ## 2.0.3
646
+ - Update deps for logstash 2.0
647
+
648
+ ## 2.0.2
649
+ - Fixes an issue when tags were defined #39
650
+
651
+ ## 2.0.0
652
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
653
+ instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
654
+ - Dependency on logstash-core update to 2.0
655
+
656
+ ## 1.0.1
657
+ - Fix a synchronization issue when doing file rotation and checking the size of the current file
658
+ - Fix an issue with synchronization when shutting down the plugin and closing the current temp file
data/CHANGELOG.md ADDED
@@ -0,0 +1,15 @@
1
+ ## 1.0.0
2
+
3
+ * Added the initial set of Logstash AWS plugins that ship with Logstash.
4
+ You can find the merged changelog of the individual plugins in CHANGELOG.PRE.MERGE.md.
5
+ These are (along with the version that was used for import):
6
+ - logstash-codec-cloudfront (3.0.3)
7
+ - logstash-codec-cloudtrail (3.0.5)
8
+ - logstash-input-cloudwatch (2.2.4)
9
+ - logstash-input-s3 (3.8.4)
10
+ - logstash-input-sqs (3.3.2)
11
+ - logstash-mixin-aws (5.1.0)
12
+ - logstash-output-cloudwatch (3.0.10)
13
+ - logstash-output-sns (4.0.8)
14
+ - logstash-output-sqs (6.0.1)
15
+ - logstash-output-s3 (4.3.5)