logstash-output-sqs 4.0.1 → 4.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +8 -1
- data/docs/index.asciidoc +218 -0
- data/logstash-output-sqs.gemspec +2 -2
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c95c261d6faf83b572387f0b6a45edf2fc7f4730
|
|
4
|
+
data.tar.gz: d44da9ac380fd7a51631a9f5e58c43747dd7c56c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d0edc16e7fc4ff65fc755b02755c26af9ea72cf757d1a94f7ed9eecd7f23017d62186b17e92be8f5e37b475048a72dd84804b7483272673f599230cbb114ecb2
|
|
7
|
+
data.tar.gz: 47af2b390102796f5741379a647f9aee0e5917e2b3c8a1537e677542f5cc8a1668b21f2071f38b82da6eef90c3d3fb6f19bc45ead82ab7f985abdacc2e3086d7
|
data/Gemfile
CHANGED
|
@@ -1,4 +1,11 @@
|
|
|
1
1
|
source 'https://rubygems.org'
|
|
2
2
|
|
|
3
|
-
# Specify your gem's dependencies in logstash-mass_effect.gemspec
|
|
4
3
|
gemspec
|
|
4
|
+
|
|
5
|
+
logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
|
|
6
|
+
use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
|
|
7
|
+
|
|
8
|
+
if Dir.exist?(logstash_path) && use_logstash_source
|
|
9
|
+
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
|
10
|
+
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
|
11
|
+
end
|
data/docs/index.asciidoc
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
:plugin: sqs
|
|
2
|
+
:type: output
|
|
3
|
+
|
|
4
|
+
///////////////////////////////////////////
|
|
5
|
+
START - GENERATED VARIABLES, DO NOT EDIT!
|
|
6
|
+
///////////////////////////////////////////
|
|
7
|
+
:version: %VERSION%
|
|
8
|
+
:release_date: %RELEASE_DATE%
|
|
9
|
+
:changelog_url: %CHANGELOG_URL%
|
|
10
|
+
:include_path: ../../../../logstash/docs/include
|
|
11
|
+
///////////////////////////////////////////
|
|
12
|
+
END - GENERATED VARIABLES, DO NOT EDIT!
|
|
13
|
+
///////////////////////////////////////////
|
|
14
|
+
|
|
15
|
+
[id="plugins-{type}-{plugin}"]
|
|
16
|
+
|
|
17
|
+
=== Sqs output plugin
|
|
18
|
+
|
|
19
|
+
include::{include_path}/plugin_header.asciidoc[]
|
|
20
|
+
|
|
21
|
+
==== Description
|
|
22
|
+
|
|
23
|
+
Push events to an Amazon Web Services (AWS) Simple Queue Service (SQS) queue.
|
|
24
|
+
|
|
25
|
+
SQS is a simple, scalable queue system that is part of the Amazon Web
|
|
26
|
+
Services suite of tools. Although SQS is similar to other queuing systems
|
|
27
|
+
such as Advanced Message Queuing Protocol (AMQP), it uses a custom API and
|
|
28
|
+
requires that you have an AWS account. See http://aws.amazon.com/sqs/ for
|
|
29
|
+
more details on how SQS works, what the pricing schedule looks like and how
|
|
30
|
+
to setup a queue.
|
|
31
|
+
|
|
32
|
+
The "consumer" identity must have the following permissions on the queue:
|
|
33
|
+
|
|
34
|
+
* `sqs:GetQueueUrl`
|
|
35
|
+
* `sqs:SendMessage`
|
|
36
|
+
* `sqs:SendMessageBatch`
|
|
37
|
+
|
|
38
|
+
Typically, you should setup an IAM policy, create a user and apply the IAM
|
|
39
|
+
policy to the user. See http://aws.amazon.com/iam/ for more details on
|
|
40
|
+
setting up AWS identities. A sample policy is as follows:
|
|
41
|
+
|
|
42
|
+
[source,json]
|
|
43
|
+
{
|
|
44
|
+
"Version": "2012-10-17",
|
|
45
|
+
"Statement": [
|
|
46
|
+
{
|
|
47
|
+
"Effect": "Allow",
|
|
48
|
+
"Action": [
|
|
49
|
+
"sqs:GetQueueUrl",
|
|
50
|
+
"sqs:SendMessage",
|
|
51
|
+
"sqs:SendMessageBatch"
|
|
52
|
+
],
|
|
53
|
+
"Resource": "arn:aws:sqs:us-east-1:123456789012:my-sqs-queue"
|
|
54
|
+
}
|
|
55
|
+
]
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
==== Batch Publishing
|
|
59
|
+
This output publishes messages to SQS in batches in order to optimize event
|
|
60
|
+
throughput and increase performance. This is done using the
|
|
61
|
+
[`SendMessageBatch`](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessageBatch.html)
|
|
62
|
+
API. When publishing messages to SQS in batches, the following service limits
|
|
63
|
+
must be respected (see
|
|
64
|
+
[Limits in Amazon SQS](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/limits-messages.html)):
|
|
65
|
+
|
|
66
|
+
* The maximum allowed individual message size is 256KiB.
|
|
67
|
+
* The maximum total payload size (i.e. the sum of the sizes of all
|
|
68
|
+
individual messages within a batch) is also 256KiB.
|
|
69
|
+
|
|
70
|
+
This plugin will dynamically adjust the size of the batch published to SQS in
|
|
71
|
+
order to ensure that the total payload size does not exceed 256KiB.
|
|
72
|
+
|
|
73
|
+
WARNING: This output cannot currently handle messages larger than 256KiB. Any
|
|
74
|
+
single message exceeding this size will be dropped.
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
[id="plugins-{type}s-{plugin}-options"]
|
|
78
|
+
==== Sqs Output Configuration Options
|
|
79
|
+
|
|
80
|
+
This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
|
|
81
|
+
|
|
82
|
+
[cols="<,<,<",options="header",]
|
|
83
|
+
|=======================================================================
|
|
84
|
+
|Setting |Input type|Required
|
|
85
|
+
| <<plugins-{type}s-{plugin}-access_key_id>> |<<string,string>>|No
|
|
86
|
+
| <<plugins-{type}s-{plugin}-aws_credentials_file>> |<<string,string>>|No
|
|
87
|
+
| <<plugins-{type}s-{plugin}-batch_events>> |<<number,number>>|No
|
|
88
|
+
| <<plugins-{type}s-{plugin}-message_max_size>> |<<bytes,bytes>>|No
|
|
89
|
+
| <<plugins-{type}s-{plugin}-proxy_uri>> |<<string,string>>|No
|
|
90
|
+
| <<plugins-{type}s-{plugin}-queue>> |<<string,string>>|Yes
|
|
91
|
+
| <<plugins-{type}s-{plugin}-region>> |<<string,string>>, one of `["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-central-1", "eu-west-1", "eu-west-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2", "sa-east-1", "us-gov-west-1", "cn-north-1", "ap-south-1", "ca-central-1"]`|No
|
|
92
|
+
| <<plugins-{type}s-{plugin}-secret_access_key>> |<<string,string>>|No
|
|
93
|
+
| <<plugins-{type}s-{plugin}-session_token>> |<<string,string>>|No
|
|
94
|
+
|=======================================================================
|
|
95
|
+
|
|
96
|
+
Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
|
|
97
|
+
output plugins.
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
[id="plugins-{type}s-{plugin}-access_key_id"]
|
|
102
|
+
===== `access_key_id`
|
|
103
|
+
|
|
104
|
+
* Value type is <<string,string>>
|
|
105
|
+
* There is no default value for this setting.
|
|
106
|
+
|
|
107
|
+
This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order:
|
|
108
|
+
|
|
109
|
+
1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config
|
|
110
|
+
2. External credentials file specified by `aws_credentials_file`
|
|
111
|
+
3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
|
|
112
|
+
4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
|
|
113
|
+
5. IAM Instance Profile (available when running inside EC2)
|
|
114
|
+
|
|
115
|
+
[id="plugins-{type}s-{plugin}-aws_credentials_file"]
|
|
116
|
+
===== `aws_credentials_file`
|
|
117
|
+
|
|
118
|
+
* Value type is <<string,string>>
|
|
119
|
+
* There is no default value for this setting.
|
|
120
|
+
|
|
121
|
+
Path to YAML file containing a hash of AWS credentials.
|
|
122
|
+
This file will only be loaded if `access_key_id` and
|
|
123
|
+
`secret_access_key` aren't set. The contents of the
|
|
124
|
+
file should look like this:
|
|
125
|
+
|
|
126
|
+
[source,ruby]
|
|
127
|
+
----------------------------------
|
|
128
|
+
:access_key_id: "12345"
|
|
129
|
+
:secret_access_key: "54321"
|
|
130
|
+
----------------------------------
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
[id="plugins-{type}s-{plugin}-batch"]
|
|
134
|
+
===== `batch` (DEPRECATED)
|
|
135
|
+
|
|
136
|
+
* DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
|
|
137
|
+
* Value type is <<boolean,boolean>>
|
|
138
|
+
* Default value is `true`
|
|
139
|
+
|
|
140
|
+
Set to `true` to send messages to SQS in batches (with the
|
|
141
|
+
`SendMessageBatch` API) or `false` to send messages to SQS individually
|
|
142
|
+
(with the `SendMessage` API). The size of the batch is configurable via
|
|
143
|
+
`batch_events`.
|
|
144
|
+
|
|
145
|
+
[id="plugins-{type}s-{plugin}-batch_events"]
|
|
146
|
+
===== `batch_events`
|
|
147
|
+
|
|
148
|
+
* Value type is <<number,number>>
|
|
149
|
+
* Default value is `10`
|
|
150
|
+
|
|
151
|
+
The number of events to be sent in each batch. Set this to `1` to disable
|
|
152
|
+
the batch sending of messages.
|
|
153
|
+
|
|
154
|
+
[id="plugins-{type}s-{plugin}-batch_timeout"]
|
|
155
|
+
===== `batch_timeout` (DEPRECATED)
|
|
156
|
+
|
|
157
|
+
* DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
|
|
158
|
+
* Value type is <<number,number>>
|
|
159
|
+
* There is no default value for this setting.
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
[id="plugins-{type}s-{plugin}-message_max_size"]
|
|
164
|
+
===== `message_max_size`
|
|
165
|
+
|
|
166
|
+
* Value type is <<bytes,bytes>>
|
|
167
|
+
* Default value is `"256KiB"`
|
|
168
|
+
|
|
169
|
+
The maximum number of bytes for any message sent to SQS. Messages exceeding
|
|
170
|
+
this size will be dropped. See
|
|
171
|
+
http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/limits-messages.html.
|
|
172
|
+
|
|
173
|
+
[id="plugins-{type}s-{plugin}-proxy_uri"]
|
|
174
|
+
===== `proxy_uri`
|
|
175
|
+
|
|
176
|
+
* Value type is <<string,string>>
|
|
177
|
+
* There is no default value for this setting.
|
|
178
|
+
|
|
179
|
+
URI to proxy server if required
|
|
180
|
+
|
|
181
|
+
[id="plugins-{type}s-{plugin}-queue"]
|
|
182
|
+
===== `queue`
|
|
183
|
+
|
|
184
|
+
* This is a required setting.
|
|
185
|
+
* Value type is <<string,string>>
|
|
186
|
+
* There is no default value for this setting.
|
|
187
|
+
|
|
188
|
+
The name of the target SQS queue. Note that this is just the name of the
|
|
189
|
+
queue, not the URL or ARN.
|
|
190
|
+
|
|
191
|
+
[id="plugins-{type}s-{plugin}-region"]
|
|
192
|
+
===== `region`
|
|
193
|
+
|
|
194
|
+
* Value can be any of: `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `eu-central-1`, `eu-west-1`, `eu-west-2`, `ap-southeast-1`, `ap-southeast-2`, `ap-northeast-1`, `ap-northeast-2`, `sa-east-1`, `us-gov-west-1`, `cn-north-1`, `ap-south-1`, `ca-central-1`
|
|
195
|
+
* Default value is `"us-east-1"`
|
|
196
|
+
|
|
197
|
+
The AWS Region
|
|
198
|
+
|
|
199
|
+
[id="plugins-{type}s-{plugin}-secret_access_key"]
|
|
200
|
+
===== `secret_access_key`
|
|
201
|
+
|
|
202
|
+
* Value type is <<string,string>>
|
|
203
|
+
* There is no default value for this setting.
|
|
204
|
+
|
|
205
|
+
The AWS Secret Access Key
|
|
206
|
+
|
|
207
|
+
[id="plugins-{type}s-{plugin}-session_token"]
|
|
208
|
+
===== `session_token`
|
|
209
|
+
|
|
210
|
+
* Value type is <<string,string>>
|
|
211
|
+
* There is no default value for this setting.
|
|
212
|
+
|
|
213
|
+
The AWS Session token for temporary credential
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
[id="plugins-{type}s-{plugin}-common-options"]
|
|
218
|
+
include::{include_path}/{type}.asciidoc[]
|
data/logstash-output-sqs.gemspec
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
|
|
3
3
|
s.name = 'logstash-output-sqs'
|
|
4
|
-
s.version = '4.0.
|
|
4
|
+
s.version = '4.0.2'
|
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
|
6
6
|
s.summary = "Push events to an Amazon Web Services Simple Queue Service (SQS) queue."
|
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
|
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
|
|
|
11
11
|
s.require_paths = ["lib"]
|
|
12
12
|
|
|
13
13
|
# Files
|
|
14
|
-
s.files = Dir[
|
|
14
|
+
s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
|
|
15
15
|
|
|
16
16
|
# Tests
|
|
17
17
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: logstash-output-sqs
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 4.0.
|
|
4
|
+
version: 4.0.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Elastic
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2017-
|
|
11
|
+
date: 2017-06-23 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -84,6 +84,7 @@ files:
|
|
|
84
84
|
- LICENSE
|
|
85
85
|
- NOTICE.TXT
|
|
86
86
|
- README.md
|
|
87
|
+
- docs/index.asciidoc
|
|
87
88
|
- lib/logstash/outputs/sqs.rb
|
|
88
89
|
- lib/logstash/outputs/sqs/patch.rb
|
|
89
90
|
- logstash-output-sqs.gemspec
|