fluent-plugin-protobuf-http 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +48 -0
- data/.rubocop.yml +19 -0
- data/Gemfile +2 -0
- data/README.md +170 -91
- data/Rakefile +2 -0
- data/fluent-plugin-protobuf-http.gemspec +5 -2
- data/lib/fluent/plugin/in_protobuf_http.rb +21 -23
- data/test/data/log.bin +3 -0
- data/test/data/log.json +1 -0
- data/test/data/logbatch5.bin +12 -0
- data/test/data/protos/log.proto +31 -0
- data/test/helper.rb +10 -5
- data/test/plugin/test_in_protobuf_http.rb +140 -6
- metadata +40 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c12d1da7ec8d1e801e5c9bc01ae3258e4c13ee6d361431721305e8a3778fe0d5
|
|
4
|
+
data.tar.gz: 2571d4e0c3795fd37c87d07fcbfd80fc7179b7bab0c7d6760c07b3fc7a561e73
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: cfb5aba44bec8f2a9ef10514ef50f5fc2d14084ccd16a8b8bb8d25babc82f351b9bed58ee3c290e00af11fb6dbcb06f4c96d41c342ee67c33fd8b42f6db1399c
|
|
7
|
+
data.tar.gz: e5e01488511a1509becf3f636e1ab0e653e243dbb44516279555963ed03b09d3333a92d9e8af07a9b9494f75b285adb97e8c7b3ac1f270111b50359054702f80
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
paths-ignore:
|
|
6
|
+
- '**.md'
|
|
7
|
+
- '.rubocop.yml'
|
|
8
|
+
pull_request:
|
|
9
|
+
paths-ignore:
|
|
10
|
+
- '**.md'
|
|
11
|
+
- '.rubocop.yml'
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
run-tests:
|
|
15
|
+
name: Run tests
|
|
16
|
+
strategy:
|
|
17
|
+
matrix:
|
|
18
|
+
os: [ubuntu-18.04]
|
|
19
|
+
ruby-version: ['2.5', '2.6', '2.7']
|
|
20
|
+
|
|
21
|
+
runs-on: ${{ matrix.os }}
|
|
22
|
+
|
|
23
|
+
steps:
|
|
24
|
+
- uses: actions/checkout@v2
|
|
25
|
+
|
|
26
|
+
- name: Set up Ruby
|
|
27
|
+
uses: ruby/setup-ruby@v1
|
|
28
|
+
with:
|
|
29
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
30
|
+
bundler-cache: true
|
|
31
|
+
|
|
32
|
+
- name: Bundle install
|
|
33
|
+
run: |
|
|
34
|
+
gem install bundler && \
|
|
35
|
+
bundle install --jobs 4 --retry 3
|
|
36
|
+
|
|
37
|
+
- name: Install protoc
|
|
38
|
+
run: |
|
|
39
|
+
mkdir protoc && cd protoc
|
|
40
|
+
wget https://github.com/protocolbuffers/protobuf/releases/download/v3.17.0/protoc-3.17.0-linux-x86_64.zip
|
|
41
|
+
unzip protoc-3.17.0-linux-x86_64.zip
|
|
42
|
+
chmod +x ./bin/protoc && sudo mv ./bin/protoc /usr/local/bin/protoc
|
|
43
|
+
sudo mv ./include/* /usr/local/include/
|
|
44
|
+
cd .. && rm -rf ./protoc
|
|
45
|
+
protoc --version
|
|
46
|
+
|
|
47
|
+
- name: Tests
|
|
48
|
+
run: bundle exec rake test
|
data/.rubocop.yml
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# RuboCop Default Config: https://github.com/rubocop-hq/rubocop/blob/master/config/default.yml
|
|
2
|
+
|
|
3
|
+
AllCops:
|
|
4
|
+
NewCops: disable
|
|
5
|
+
|
|
6
|
+
Gemspec/RequiredRubyVersion:
|
|
7
|
+
Enabled: false
|
|
8
|
+
|
|
9
|
+
Metrics/ClassLength:
|
|
10
|
+
Enabled: false
|
|
11
|
+
|
|
12
|
+
Metrics/MethodLength:
|
|
13
|
+
Enabled: false
|
|
14
|
+
|
|
15
|
+
Metrics/AbcSize:
|
|
16
|
+
Enabled: false
|
|
17
|
+
|
|
18
|
+
Layout/LineLength:
|
|
19
|
+
Enabled: false
|
data/Gemfile
CHANGED
data/README.md
CHANGED
|
@@ -1,62 +1,79 @@
|
|
|
1
1
|
# fluent-plugin-protobuf-http
|
|
2
2
|
|
|
3
|
+
[](https://github.com/iamazeem/fluent-plugin-protobuf-http/actions/workflows/ci.yml)
|
|
4
|
+
[](https://github.com/iamAzeem/fluent-plugin-protobuf-http/blob/master/LICENSE)
|
|
5
|
+
|
|
6
|
+
[](https://rubygems.org/gems/fluent-plugin-protobuf-http)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
3
12
|
[Fluentd](https://fluentd.org/) HTTP input plugin for Protocol Buffers.
|
|
4
13
|
|
|
5
14
|
## Features
|
|
6
15
|
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
|
|
10
|
-
*
|
|
11
|
-
*
|
|
16
|
+
* Automatic compilation of `.proto` files located in `proto_dir`
|
|
17
|
+
* Incoming Format: Binary or JSON (`Content-Type`: `application/octet-stream` or
|
|
18
|
+
`application/json`)
|
|
19
|
+
* Outgoing Format: Binary or JSON
|
|
20
|
+
* Single and Batch message support
|
|
21
|
+
* TLS Support with `<transport>` section and `https://` URL protocol prefix.
|
|
22
|
+
|
|
23
|
+
For more details on TLS configuration, see this official
|
|
24
|
+
[example](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-server#configuration-example).
|
|
12
25
|
|
|
13
26
|
## Installation
|
|
14
27
|
|
|
15
28
|
### RubyGems
|
|
16
29
|
|
|
17
|
-
```
|
|
18
|
-
|
|
30
|
+
```shell
|
|
31
|
+
gem install fluent-plugin-protobuf-http
|
|
19
32
|
```
|
|
20
33
|
|
|
21
34
|
### Bundler
|
|
22
35
|
|
|
23
36
|
Add following line to your Gemfile:
|
|
37
|
+
|
|
24
38
|
```ruby
|
|
25
|
-
gem
|
|
39
|
+
gem 'fluent-plugin-protobuf-http'
|
|
26
40
|
```
|
|
27
41
|
|
|
28
42
|
And then execute:
|
|
29
|
-
|
|
30
|
-
|
|
43
|
+
|
|
44
|
+
```shell
|
|
45
|
+
bundle
|
|
31
46
|
```
|
|
32
47
|
|
|
33
48
|
## Configuration
|
|
34
49
|
|
|
35
|
-
*
|
|
36
|
-
* Default
|
|
37
|
-
*
|
|
38
|
-
* Default
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
* Default
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
* Default
|
|
46
|
-
*
|
|
47
|
-
|
|
48
|
-
###
|
|
49
|
-
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
* Default
|
|
53
|
-
*
|
|
50
|
+
* `bind` (string) (optional): The address to listen to.
|
|
51
|
+
* Default: `0.0.0.0`
|
|
52
|
+
* `port` (integer) (optional): The port to listen to.
|
|
53
|
+
* Default: `8080`
|
|
54
|
+
* `proto_dir` (string) (required): The directory path that contains the .proto files.
|
|
55
|
+
* `in_mode` (enum) (optional): The mode of incoming (supported) events.
|
|
56
|
+
* Modes: `binary`, `json`
|
|
57
|
+
* Default: `binary`
|
|
58
|
+
* `out_mode` (enum) (optional): The mode of outgoing (emitted) events.
|
|
59
|
+
* Modes: `binary`, `json`
|
|
60
|
+
* Default: `binary`
|
|
61
|
+
* `tag` (string) (required): The tag for the event.
|
|
62
|
+
|
|
63
|
+
### `<transport>` section (optional) (single)
|
|
64
|
+
|
|
65
|
+
* `protocol` (enum) (optional):
|
|
66
|
+
* Protocols: `tcp`, `tls`
|
|
67
|
+
* Default: `tcp`
|
|
68
|
+
* For more details, see this official configuration
|
|
69
|
+
[example](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-server#configuration-example).
|
|
54
70
|
|
|
55
71
|
### Example
|
|
56
72
|
|
|
57
|
-
```
|
|
58
|
-
#
|
|
59
|
-
#
|
|
73
|
+
```text
|
|
74
|
+
# Endpoints:
|
|
75
|
+
# - Single Message: http://ip:port/<tag>?msgtype=<msgtype>
|
|
76
|
+
# - Batch Message: http://ip:port/<tag>?msgtype=<batch-msgtype>?batch=true
|
|
60
77
|
|
|
61
78
|
<source>
|
|
62
79
|
@type protobuf_http
|
|
@@ -74,12 +91,18 @@ $ bundle
|
|
|
74
91
|
|
|
75
92
|
## Schemas (`.proto` files)
|
|
76
93
|
|
|
77
|
-
The
|
|
78
|
-
|
|
79
|
-
|
|
94
|
+
The prime use-case for this plugin is assumed to be event logging. So, always
|
|
95
|
+
use self-contained `.proto` file(s) that do not import other `.proto` files. The
|
|
96
|
+
names e.g. `package`, `message`, etc. must be unique and are treated as
|
|
97
|
+
case-sensitive.
|
|
80
98
|
|
|
81
|
-
Consider this
|
|
82
|
-
|
|
99
|
+
Consider this
|
|
100
|
+
[log.proto](https://github.com/iamAzeem/protobuf-log-sample/blob/master/log.proto)
|
|
101
|
+
schema from
|
|
102
|
+
[protobuf-log-sample](https://github.com/iamAzeem/protobuf-log-sample)
|
|
103
|
+
repository:
|
|
104
|
+
|
|
105
|
+
```protobuf
|
|
83
106
|
syntax = "proto3";
|
|
84
107
|
|
|
85
108
|
package service.logging;
|
|
@@ -108,9 +131,9 @@ message Log {
|
|
|
108
131
|
}
|
|
109
132
|
```
|
|
110
133
|
|
|
111
|
-
The fully-qualified message type for `Log`
|
|
112
|
-
|
|
113
|
-
|
|
134
|
+
The fully-qualified message type for `Log` is `service.logging.Log`. This
|
|
135
|
+
message type is used as the value of `msgtype` query parameter in the URL. See
|
|
136
|
+
URL section below for more on `msgtype`.
|
|
114
137
|
|
|
115
138
|
### Single Message
|
|
116
139
|
|
|
@@ -118,20 +141,21 @@ The above schema will be used as-is for the single message.
|
|
|
118
141
|
|
|
119
142
|
### Batch Message
|
|
120
143
|
|
|
121
|
-
For
|
|
122
|
-
|
|
144
|
+
For the batch message, the schema must be like this:
|
|
145
|
+
|
|
146
|
+
```protobuf
|
|
123
147
|
message Batch {
|
|
124
148
|
string type = 1;
|
|
125
149
|
repeated Log batch = 2;
|
|
126
150
|
}
|
|
127
151
|
```
|
|
128
152
|
|
|
129
|
-
IMPORTANT:
|
|
130
|
-
|
|
131
|
-
You can choose any name for a batch message type.
|
|
153
|
+
IMPORTANT: The `Batch` message type is part of `log.proto`, it is not a separate
|
|
154
|
+
file! You can choose any name for a batch message type.
|
|
132
155
|
|
|
133
|
-
|
|
134
|
-
|
|
156
|
+
Here is the complete `log.proto` file:
|
|
157
|
+
|
|
158
|
+
```protobuf
|
|
135
159
|
syntax = "proto3";
|
|
136
160
|
|
|
137
161
|
package service.logging;
|
|
@@ -165,50 +189,62 @@ message Batch {
|
|
|
165
189
|
}
|
|
166
190
|
```
|
|
167
191
|
|
|
168
|
-
For batch processing, the plugin looks for special members `type` and `batch`.
|
|
192
|
+
For batch processing, the plugin looks for special members `type` and `batch`.
|
|
169
193
|
The `type` will indicate the message type of `batch` i.e. `Log` in this example.
|
|
170
194
|
|
|
171
|
-
The type of `Batch` is `service.logging.Batch` and it will be the value of
|
|
172
|
-
The type of `batch` array is `service.logging.Log`
|
|
195
|
+
The type of `Batch` is `service.logging.Batch` and it will be the value of
|
|
196
|
+
`msgtype` in the URL query. The type of `batch` array is `service.logging.Log`
|
|
197
|
+
and it will be the value of `type`.
|
|
173
198
|
|
|
174
|
-
The `google.protobuf.Any` type has not been used deliberately
|
|
175
|
-
|
|
176
|
-
|
|
199
|
+
The `google.protobuf.Any` type has not been used here deliberately. It stores
|
|
200
|
+
the message type with each message resulting in an increase in size. Refer to
|
|
201
|
+
[protobuf-repeated-type-vs-any](https://github.com/iamAzeem/protobuf-repeated-type-vs-any)
|
|
202
|
+
for a simple comparison. With the above approach, the type is stored only once
|
|
203
|
+
for the whole batch.
|
|
177
204
|
|
|
178
205
|
### Endpoint (URL)
|
|
179
206
|
|
|
180
207
|
For single message:
|
|
181
|
-
|
|
208
|
+
|
|
209
|
+
```text
|
|
182
210
|
http://<ip>:<port>/<tag>?msgtype=<fully-qualified-message-type>
|
|
183
211
|
```
|
|
184
212
|
|
|
185
213
|
For batch message:
|
|
186
|
-
|
|
214
|
+
|
|
215
|
+
```text
|
|
187
216
|
http://<ip>:<port>/<tag>?msgtype=<fully-qualified-message-type-for-batch>&batch=true
|
|
188
217
|
```
|
|
189
218
|
|
|
190
|
-
Without `batch=true` query parameter, the batch will be treated as a single
|
|
219
|
+
Without `batch=true` query parameter, the batch will be treated as a single
|
|
220
|
+
message.
|
|
191
221
|
|
|
192
|
-
For example, for a log type `service.logging.Log` and its corresponding batch
|
|
222
|
+
For example, for a log type `service.logging.Log` and its corresponding batch
|
|
223
|
+
type `service.logging.Batch`, the URLs would be:
|
|
193
224
|
|
|
194
|
-
|
|
195
|
-
|
|
225
|
+
For single message:
|
|
226
|
+
|
|
227
|
+
```text
|
|
196
228
|
http://localhost:8080/debug.test?msgtype=service.logging.Log
|
|
197
229
|
```
|
|
198
230
|
|
|
199
|
-
|
|
200
|
-
|
|
231
|
+
For batch message:
|
|
232
|
+
|
|
233
|
+
```text
|
|
201
234
|
http://localhost:8080/debug.test?msgtype=service.logging.Batch&batch=true
|
|
202
235
|
```
|
|
203
236
|
|
|
204
|
-
**NOTE**: The values of query parameters (`msgtype`, `batch`) are
|
|
237
|
+
**NOTE**: The values of query parameters (`msgtype`, `batch`) are
|
|
238
|
+
case-sensitive!
|
|
205
239
|
|
|
206
240
|
## Test Use-Case (`curl`)
|
|
207
241
|
|
|
208
|
-
For a simple
|
|
242
|
+
For a simple use-case of incoming HTTP events and their routing to
|
|
243
|
+
[stdout](https://docs.fluentd.org/output/stdout) may be configured like this:
|
|
209
244
|
|
|
210
245
|
`fluent.conf`:
|
|
211
|
-
|
|
246
|
+
|
|
247
|
+
```text
|
|
212
248
|
<source>
|
|
213
249
|
@type protobuf_http
|
|
214
250
|
@id protobuf_http_input
|
|
@@ -224,24 +260,36 @@ For a simple test use-case of events and their routing to [stdout](https://docs.
|
|
|
224
260
|
|
|
225
261
|
<match debug.test>
|
|
226
262
|
@type stdout
|
|
227
|
-
@id stdout_output
|
|
228
263
|
</match>
|
|
229
264
|
```
|
|
230
265
|
|
|
231
266
|
The incoming binary messages will be transformed to JSON for further consumption.
|
|
232
267
|
|
|
233
|
-
|
|
268
|
+
### Single Message
|
|
269
|
+
|
|
270
|
+
Test Parameters:
|
|
234
271
|
|
|
235
|
-
|
|
236
|
-
|
|
272
|
+
| input file | single message type |
|
|
273
|
+
|:----------:|:---------------------:|
|
|
274
|
+
| `log.bin` | `service.logging.Log` |
|
|
237
275
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
276
|
+
URL:
|
|
277
|
+
|
|
278
|
+
```bash
|
|
279
|
+
http://localhost:8080/debug.test?msgtype=service.logging.Log
|
|
241
280
|
```
|
|
242
281
|
|
|
243
|
-
`
|
|
282
|
+
`curl` command:
|
|
283
|
+
|
|
284
|
+
```shell
|
|
285
|
+
curl -X POST -H "Content-Type: application/octet-stream" \
|
|
286
|
+
--data-binary "@/<path>/log.bin" \
|
|
287
|
+
"http://localhost:8080/debug.test?msgtype=service.logging.Log"
|
|
244
288
|
```
|
|
289
|
+
|
|
290
|
+
`fluentd` logs (Observe JSON at the end):
|
|
291
|
+
|
|
292
|
+
```text
|
|
245
293
|
2020-06-09 18:53:47 +0500 [info]: #0 [protobuf_http_input] [R] {binary} [127.0.0.1:41222, size: 86 bytes]
|
|
246
294
|
2020-06-09 18:53:47 +0500 [warn]: #0 [protobuf_http_input] 'batch' not found in 'query_string' [msgtype=service.logging.Log]
|
|
247
295
|
2020-06-09 18:53:47 +0500 [info]: #0 [protobuf_http_input] [S] {binary} [127.0.0.1:41222, msgtype: service.logging.Log, size: 86 bytes]
|
|
@@ -249,22 +297,38 @@ $ curl -X POST -H "Content-Type: application/octet-stream" --data-binary "@/<pat
|
|
|
249
297
|
2020-06-09 18:53:47 +0500 [info]: #0 [protobuf_http_input] [S] {json} [127.0.0.1:41222, msgtype: service.logging.Log, size: 183 bytes]
|
|
250
298
|
```
|
|
251
299
|
|
|
252
|
-
For
|
|
300
|
+
For sample Single message generation, see
|
|
301
|
+
[this](https://github.com/iamAzeem/protobuf-log-sample).
|
|
302
|
+
|
|
303
|
+
### Batch Message
|
|
304
|
+
|
|
305
|
+
Test Parameters:
|
|
253
306
|
|
|
254
|
-
|
|
307
|
+
| input file | batch message type | batch internal type | messages |
|
|
308
|
+
|:---------------:|:-----------------------:|:---------------------:|:--------:|
|
|
309
|
+
| `logbatch2.bin` | `service.logging.Batch` | `service.logging.Log` | 2 |
|
|
310
|
+
| `logbatch5.bin` | `service.logging.Batch` | `service.logging.Log` | 5 |
|
|
255
311
|
|
|
256
|
-
|
|
257
|
-
* data: `logbatch2.bin`, msgtype: `service.logging.Batch`, type: `service.logging.Log` [batch_size: 2 messages]
|
|
258
|
-
* data: `logbatch5.bin`, msgtype: `service.logging.Batch`, type: `service.logging.Log` [batch_size: 5 messages]
|
|
312
|
+
URL:
|
|
259
313
|
|
|
260
|
-
|
|
314
|
+
```text
|
|
315
|
+
http://localhost:8080/debug.test?msgtype=service.logging.Batch&batch=true
|
|
261
316
|
```
|
|
262
|
-
|
|
317
|
+
|
|
318
|
+
**`logbatch2.bin`**
|
|
319
|
+
|
|
320
|
+
`curl` command:
|
|
321
|
+
|
|
322
|
+
```shell
|
|
323
|
+
$ curl -X POST -H "Content-Type: application/octet-stream" \
|
|
324
|
+
--data-binary "@/<path>/logbatch2.bin" \
|
|
325
|
+
"http://localhost:8080/debug.test?msgtype=service.logging.Batch&batch=true"
|
|
263
326
|
{"status":"Batch received! [batch_type: service.logging.Log, batch_size: 2 messages]"}
|
|
264
327
|
```
|
|
265
328
|
|
|
266
|
-
`fluentd`
|
|
267
|
-
|
|
329
|
+
`fluentd` logs:
|
|
330
|
+
|
|
331
|
+
```text
|
|
268
332
|
2020-06-09 19:04:13 +0500 [info]: #0 [protobuf_http_input] [R] {binary} [127.0.0.1:41416, size: 207 bytes]
|
|
269
333
|
2020-06-09 19:04:13 +0500 [info]: #0 [protobuf_http_input] [B] {binary} [127.0.0.1:41416, msgtype: service.logging.Batch, size: 207 bytes]
|
|
270
334
|
2020-06-09 19:04:13 +0500 [info]: #0 [protobuf_http_input] [B] Emitting message stream/batch [batch_size: 2 messages]...
|
|
@@ -273,14 +337,20 @@ $ curl -X POST -H "Content-Type: application/octet-stream" --data-binary "@/<pat
|
|
|
273
337
|
2020-06-09 19:04:13 +0500 [info]: #0 [protobuf_http_input] [B] {json} [127.0.0.1:41416, msgtype: service.logging.Batch] Batch received! [batch_type: service.logging.Log, batch_size: 2 messages]
|
|
274
338
|
```
|
|
275
339
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
340
|
+
**`logbatch5.bin`**
|
|
341
|
+
|
|
342
|
+
`curl` command:
|
|
343
|
+
|
|
344
|
+
```bash
|
|
345
|
+
$ curl -X POST -H "Content-Type: application/octet-stream" \
|
|
346
|
+
--data-binary "@/<path>/logbatch5.bin" \
|
|
347
|
+
"http://localhost:8080/debug.test?msgtype=service.logging.Batch&batch=true"
|
|
279
348
|
{"status":"Batch received! [batch_type: service.logging.Log, batch_size: 5 messages]"}
|
|
280
349
|
```
|
|
281
350
|
|
|
282
|
-
`fluentd`
|
|
283
|
-
|
|
351
|
+
`fluentd` logs:
|
|
352
|
+
|
|
353
|
+
```text
|
|
284
354
|
2020-06-09 19:07:09 +0500 [info]: #0 [protobuf_http_input] [R] {binary} [127.0.0.1:41552, size: 486 bytes]
|
|
285
355
|
2020-06-09 19:07:09 +0500 [info]: #0 [protobuf_http_input] [B] {binary} [127.0.0.1:41552, msgtype: service.logging.Batch, size: 486 bytes]
|
|
286
356
|
2020-06-09 19:07:09 +0500 [info]: #0 [protobuf_http_input] [B] Emitting message stream/batch [batch_size: 5 messages]...
|
|
@@ -292,10 +362,19 @@ $ curl -X POST -H "Content-Type: application/octet-stream" --data-binary "@/<pat
|
|
|
292
362
|
2020-06-09 19:07:09 +0500 [info]: #0 [protobuf_http_input] [B] {json} [127.0.0.1:41552, msgtype: service.logging.Batch] Batch received! [batch_type: service.logging.Log, batch_size: 5 messages]
|
|
293
363
|
```
|
|
294
364
|
|
|
295
|
-
For
|
|
365
|
+
For sample Batch message generation, see
|
|
366
|
+
[this](https://gist.github.com/iamAzeem/a8a24092132e1741a76956192f2104cc).
|
|
367
|
+
|
|
368
|
+
## Contribute
|
|
369
|
+
|
|
370
|
+
- Fork the project.
|
|
371
|
+
- Check out the latest `main` branch.
|
|
372
|
+
- Create a feature or bugfix branch from `main`.
|
|
373
|
+
- Commit and push your changes.
|
|
374
|
+
- Make sure to add and run tests locally: `bundle exec rake test`.
|
|
375
|
+
- Run `rubocop` locally and fix all the lint warnings.
|
|
376
|
+
- Submit the PR.
|
|
296
377
|
|
|
297
|
-
##
|
|
378
|
+
## License
|
|
298
379
|
|
|
299
|
-
|
|
300
|
-
* License
|
|
301
|
-
* Apache License, Version 2.0
|
|
380
|
+
[Apache 2.0](LICENSE)
|
data/Rakefile
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
# frozen-string-literal: true
|
|
2
|
+
|
|
1
3
|
lib = File.expand_path('../lib', __dir__)
|
|
2
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
3
5
|
|
|
4
6
|
Gem::Specification.new do |spec|
|
|
5
7
|
spec.name = 'fluent-plugin-protobuf-http'
|
|
6
|
-
spec.version = '0.
|
|
8
|
+
spec.version = '0.3.0'
|
|
7
9
|
spec.authors = ['Azeem Sajid']
|
|
8
10
|
spec.email = ['azeem.sajid@gmail.com']
|
|
9
11
|
|
|
@@ -20,8 +22,9 @@ Gem::Specification.new do |spec|
|
|
|
20
22
|
spec.test_files = test_files
|
|
21
23
|
spec.require_paths = ['lib']
|
|
22
24
|
|
|
23
|
-
spec.add_development_dependency 'bundler', '~> 1.
|
|
25
|
+
spec.add_development_dependency 'bundler', '~> 2.1', '>= 2.1.0'
|
|
24
26
|
spec.add_development_dependency 'rake', '~> 12.0'
|
|
27
|
+
spec.add_development_dependency 'simplecov', '~> 0.12', '<= 0.12.2'
|
|
25
28
|
spec.add_development_dependency 'test-unit', '~> 3.0'
|
|
26
29
|
spec.add_runtime_dependency 'fluentd', ['>= 0.14.10', '< 2']
|
|
27
30
|
spec.add_runtime_dependency 'google-protobuf', '~> 3.12', '>= 3.12.2'
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen-string-literal: true
|
|
2
|
+
|
|
1
3
|
#
|
|
2
4
|
# Copyright 2020 Azeem Sajid
|
|
3
5
|
#
|
|
@@ -18,9 +20,11 @@ require 'fluent/config/error'
|
|
|
18
20
|
require 'fluent/plugin_helper/http_server'
|
|
19
21
|
require 'webrick/httputils'
|
|
20
22
|
require 'json'
|
|
23
|
+
require 'English'
|
|
21
24
|
|
|
22
25
|
module Fluent
|
|
23
26
|
module Plugin
|
|
27
|
+
# Implementation of HTTP input plugin for Protobuf
|
|
24
28
|
class ProtobufHttpInput < Fluent::Plugin::Input
|
|
25
29
|
Fluent::Plugin.register_input('protobuf_http', self)
|
|
26
30
|
|
|
@@ -46,10 +50,6 @@ module Fluent
|
|
|
46
50
|
config_argument :protocol, :enum, list: %i[tcp tls], default: :tcp
|
|
47
51
|
end
|
|
48
52
|
|
|
49
|
-
config_section :transform, required: false, multi: false, init: true, param_name: :transform_config do
|
|
50
|
-
config_argument :msgtype, :string
|
|
51
|
-
end
|
|
52
|
-
|
|
53
53
|
def initialize
|
|
54
54
|
super
|
|
55
55
|
|
|
@@ -61,16 +61,16 @@ module Fluent
|
|
|
61
61
|
def compile_protos
|
|
62
62
|
log.debug("Checking proto_dir [#{@proto_dir}]...")
|
|
63
63
|
|
|
64
|
-
path = File.expand_path(@proto_dir)
|
|
64
|
+
path = File.expand_path(@proto_dir)
|
|
65
65
|
raise Fluent::ConfigError, "protos_dir does not exist! [#{path}]" unless Dir.exist?(path)
|
|
66
66
|
|
|
67
|
-
@protos = Dir["#{path}/*.proto"]
|
|
67
|
+
@protos = Dir["#{path}/*.proto"]
|
|
68
68
|
raise Fluent::ConfigError, "Empty proto_dir! [#{path}]" unless @protos.any?
|
|
69
69
|
|
|
70
70
|
log.info("Compiling .proto files [#{@protos.length}]...")
|
|
71
71
|
|
|
72
72
|
`protoc --ruby_out=#{path} --proto_path=#{path} #{path}/*.proto`
|
|
73
|
-
raise Fluent::ConfigError, 'Could not compile! See error(s) above.' unless
|
|
73
|
+
raise Fluent::ConfigError, 'Could not compile! See error(s) above.' unless $CHILD_STATUS.success?
|
|
74
74
|
|
|
75
75
|
log.info("Compiled successfully:\n- #{@protos.join("\n- ")}")
|
|
76
76
|
|
|
@@ -82,8 +82,8 @@ module Fluent
|
|
|
82
82
|
end
|
|
83
83
|
|
|
84
84
|
def get_compiled_proto(proto)
|
|
85
|
-
proto_suffix = '.proto'
|
|
86
|
-
compiled_proto_suffix = '_pb.rb'
|
|
85
|
+
proto_suffix = '.proto'
|
|
86
|
+
compiled_proto_suffix = '_pb.rb'
|
|
87
87
|
|
|
88
88
|
compiled_proto = proto.chomp(proto_suffix) + compiled_proto_suffix
|
|
89
89
|
raise Fluent::ConfigError, "Compiled proto not found! [#{compiled_proto}]" unless File.file?(compiled_proto)
|
|
@@ -117,7 +117,7 @@ module Fluent
|
|
|
117
117
|
msg_types = []
|
|
118
118
|
File.foreach(compiled_proto) do |line|
|
|
119
119
|
if line.lstrip.start_with?('add_message')
|
|
120
|
-
msg_type = line[/"([^"]*)"/, 1]
|
|
120
|
+
msg_type = line[/"([^"]*)"/, 1] # regex: <add_message> 'msg_type' <do>
|
|
121
121
|
msg_types.push(msg_type) unless msg_type.nil?
|
|
122
122
|
end
|
|
123
123
|
end
|
|
@@ -152,15 +152,13 @@ module Fluent
|
|
|
152
152
|
tls_opts = @transport_config.to_h
|
|
153
153
|
end
|
|
154
154
|
|
|
155
|
-
log.warn("#{@transform_config.to_h}")
|
|
156
|
-
|
|
157
155
|
log.info("Starting protobuf #{proto == :tcp ? 'HTTP' : 'HTTPS'} server [#{@bind}:#{@port}]...")
|
|
158
156
|
log.debug("TLS configuration:\n#{tls_opts}") if tls_opts
|
|
159
157
|
|
|
160
158
|
http_server_create_http_server(:protobuf_server, addr: @bind, port: @port, logger: log, proto: proto, tls_opts: tls_opts) do |server|
|
|
161
159
|
server.post("/#{tag}") do |req|
|
|
162
|
-
peeraddr = "#{req.peeraddr[2]}:#{req.peeraddr[1]}"
|
|
163
|
-
serialized_msg = req.body
|
|
160
|
+
peeraddr = "#{req.peeraddr[2]}:#{req.peeraddr[1]}" # ip:port
|
|
161
|
+
serialized_msg = req.body
|
|
164
162
|
|
|
165
163
|
log.info("[R] {#{@in_mode}} [#{peeraddr}, size: #{serialized_msg.length} bytes]")
|
|
166
164
|
log.debug("Dumping serialized message [#{serialized_msg.length} bytes]:\n#{serialized_msg}")
|
|
@@ -168,7 +166,7 @@ module Fluent
|
|
|
168
166
|
content_type = req.header['content-type'][0]
|
|
169
167
|
|
|
170
168
|
unless valid_content_type?(content_type)
|
|
171
|
-
status = "Invalid 'Content-Type' header! [#{content_type}]"
|
|
169
|
+
status = "Invalid 'Content-Type' header! [#{content_type}]"
|
|
172
170
|
log.warn("[X] Message rejected! [#{peeraddr}] #{status}")
|
|
173
171
|
next [400, { 'Content-Type' => 'application/json', 'Connection' => 'close' }, { 'status' => status }.to_json]
|
|
174
172
|
end
|
|
@@ -177,7 +175,7 @@ module Fluent
|
|
|
177
175
|
|
|
178
176
|
msgtype, batch = get_query_params(req.query_string)
|
|
179
177
|
unless @msgclass_lookup.key?(msgtype)
|
|
180
|
-
status = "Invalid 'msgtype' in 'query_string'! [#{msgtype}]"
|
|
178
|
+
status = "Invalid 'msgtype' in 'query_string'! [#{msgtype}]"
|
|
181
179
|
log.warn("[X] Message rejected! [#{peeraddr}] #{status}")
|
|
182
180
|
next [400, { 'Content-Type' => 'application/json', 'Connection' => 'close' }, { 'status' => status }.to_json]
|
|
183
181
|
end
|
|
@@ -187,7 +185,7 @@ module Fluent
|
|
|
187
185
|
deserialized_msg = deserialize_msg(msgtype, serialized_msg)
|
|
188
186
|
|
|
189
187
|
if deserialized_msg.nil?
|
|
190
|
-
status = "Incompatible message! [msgtype: #{msgtype}, size: #{serialized_msg.length} bytes]"
|
|
188
|
+
status = "Incompatible message! [msgtype: #{msgtype}, size: #{serialized_msg.length} bytes]"
|
|
191
189
|
log.warn("[X] Message rejected! [#{peeraddr}] #{status}")
|
|
192
190
|
next [400, { 'Content-Type' => 'application/json', 'Connection' => 'close' }, { 'status' => status }.to_json]
|
|
193
191
|
end
|
|
@@ -214,7 +212,7 @@ module Fluent
|
|
|
214
212
|
log.info("[B] {#{@in_mode}} [#{peeraddr}, msgtype: #{msgtype}, size: #{serialized_msg.length} bytes]")
|
|
215
213
|
|
|
216
214
|
if deserialized_msg.type.nil? || deserialized_msg.batch.nil? || deserialized_msg.batch.empty?
|
|
217
|
-
status = "Invalid 'batch' message! [msgtype: #{msgtype}, size: #{serialized_msg.length} bytes]"
|
|
215
|
+
status = "Invalid 'batch' message! [msgtype: #{msgtype}, size: #{serialized_msg.length} bytes]"
|
|
218
216
|
log.warn("[X] Message rejected! [#{peeraddr}] #{status}")
|
|
219
217
|
next [400, { 'Content-Type' => 'application/json', 'Connection' => 'close' }, { 'status' => status }.to_json]
|
|
220
218
|
end
|
|
@@ -234,7 +232,7 @@ module Fluent
|
|
|
234
232
|
|
|
235
233
|
router.emit_stream(@tag, stream)
|
|
236
234
|
|
|
237
|
-
status = "Batch received! [batch_type: #{batch_type}, batch_size: #{batch_size} messages]"
|
|
235
|
+
status = "Batch received! [batch_type: #{batch_type}, batch_size: #{batch_size} messages]"
|
|
238
236
|
log.info("[B] {#{@out_mode}} [#{peeraddr}, msgtype: #{msgtype}] #{status}")
|
|
239
237
|
[200, { 'Content-Type' => 'application/json', 'Connection' => 'close' }, { 'status' => status }.to_json]
|
|
240
238
|
end
|
|
@@ -242,8 +240,8 @@ module Fluent
|
|
|
242
240
|
end
|
|
243
241
|
|
|
244
242
|
def valid_content_type?(content_type)
|
|
245
|
-
hdr_binary = 'application/octet-stream'
|
|
246
|
-
hdr_json = 'application/json'
|
|
243
|
+
hdr_binary = 'application/octet-stream'
|
|
244
|
+
hdr_json = 'application/json'
|
|
247
245
|
|
|
248
246
|
case @in_mode
|
|
249
247
|
when :binary
|
|
@@ -284,7 +282,7 @@ module Fluent
|
|
|
284
282
|
rescue Google::Protobuf::ParseError => e
|
|
285
283
|
log.error("Incompatible message! [msgtype: #{msgtype}, size: #{serialized_msg.length} bytes] #{e}")
|
|
286
284
|
nil
|
|
287
|
-
rescue => e
|
|
285
|
+
rescue StandardError => e
|
|
288
286
|
log.error("Deserializaton failed! Error: #{e}")
|
|
289
287
|
nil
|
|
290
288
|
end
|
|
@@ -300,7 +298,7 @@ module Fluent
|
|
|
300
298
|
when :json
|
|
301
299
|
msgclass.encode_json(deserialized_msg)
|
|
302
300
|
end
|
|
303
|
-
rescue => e
|
|
301
|
+
rescue StandardError => e
|
|
304
302
|
log.error("Serialization failed! [msgtype: #{msgtype}, msg: #{deserialized_msg}] Error: #{e}")
|
|
305
303
|
nil
|
|
306
304
|
end
|
data/test/data/log.bin
ADDED
data/test/data/log.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"context":{"timestamp":"2020-06-01T16:24:19Z","hostOrIp":"192.168.xxx.xxx","serviceName":"test","user":"test"},"level":"INFO","message":"This is a test log generated by [./log.rb]."}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
|
|
2
|
+
service.logging.Log[
|
|
3
|
+
%
|
|
4
|
+
����192.168.xxx.xxxtest"test0This is a test log generated by [./logbatch.rb].[
|
|
5
|
+
%
|
|
6
|
+
����192.168.xxx.xxxtest"test0This is a test log generated by [./logbatch.rb].[
|
|
7
|
+
%
|
|
8
|
+
����192.168.xxx.xxxtest"test0This is a test log generated by [./logbatch.rb].[
|
|
9
|
+
%
|
|
10
|
+
����192.168.xxx.xxxtest"test0This is a test log generated by [./logbatch.rb].[
|
|
11
|
+
%
|
|
12
|
+
����192.168.xxx.xxxtest"test0This is a test log generated by [./logbatch.rb].
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
syntax = "proto3";
|
|
2
|
+
|
|
3
|
+
package service.logging;
|
|
4
|
+
|
|
5
|
+
import "google/protobuf/timestamp.proto";
|
|
6
|
+
|
|
7
|
+
message Log {
|
|
8
|
+
message Context {
|
|
9
|
+
google.protobuf.Timestamp timestamp = 1;
|
|
10
|
+
string host_or_ip = 2;
|
|
11
|
+
string service_name = 3;
|
|
12
|
+
string user = 4;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
enum Level {
|
|
16
|
+
DEBUG = 0;
|
|
17
|
+
INFO = 1;
|
|
18
|
+
WARN = 2;
|
|
19
|
+
ERROR = 3;
|
|
20
|
+
FATAL = 4;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
Context context = 1;
|
|
24
|
+
Level level = 2;
|
|
25
|
+
string message = 3;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
message Batch {
|
|
29
|
+
string type = 1;
|
|
30
|
+
repeated Log batch = 2;
|
|
31
|
+
}
|
data/test/helper.rb
CHANGED
|
@@ -1,8 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
require
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
# frozen-string-literal: true
|
|
2
|
+
|
|
3
|
+
require 'simplecov'
|
|
4
|
+
SimpleCov.start
|
|
5
|
+
|
|
6
|
+
$LOAD_PATH.unshift(File.expand_path('..', __dir__))
|
|
7
|
+
require 'test-unit'
|
|
8
|
+
require 'fluent/test'
|
|
9
|
+
require 'fluent/test/driver/input'
|
|
10
|
+
require 'fluent/test/helpers'
|
|
6
11
|
|
|
7
12
|
Test::Unit::TestCase.include(Fluent::Test::Helpers)
|
|
8
13
|
Test::Unit::TestCase.extend(Fluent::Test::Helpers)
|
|
@@ -1,18 +1,152 @@
|
|
|
1
|
-
|
|
2
|
-
require "fluent/plugin/in_protobuf_http.rb"
|
|
1
|
+
# frozen-string-literal: true
|
|
3
2
|
|
|
3
|
+
require 'helper'
|
|
4
|
+
require 'fluent/plugin/in_protobuf_http'
|
|
5
|
+
require 'net/http'
|
|
6
|
+
|
|
7
|
+
# Implementation of Test Class
|
|
4
8
|
class ProtobufHttpInputTest < Test::Unit::TestCase
|
|
5
9
|
setup do
|
|
6
10
|
Fluent::Test.setup
|
|
11
|
+
@log_bin = File.open('./test/data/log.bin', 'rb') { |f| f.read }
|
|
12
|
+
@log_json = File.open('./test/data/log.json', 'r') { |f| f.read }
|
|
13
|
+
@log_bin_batch = File.open('./test/data/logbatch5.bin', 'rb') { |f| f.read }
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def create_driver(conf)
|
|
17
|
+
Fluent::Test::Driver::Input.new(Fluent::Plugin::ProtobufHttpInput).configure(conf)
|
|
7
18
|
end
|
|
8
19
|
|
|
9
|
-
|
|
10
|
-
|
|
20
|
+
sub_test_case 'configure' do
|
|
21
|
+
test 'test default configuration' do
|
|
22
|
+
conf = %(
|
|
23
|
+
proto_dir ./test/data/protos
|
|
24
|
+
tag test
|
|
25
|
+
)
|
|
26
|
+
driver = create_driver(conf)
|
|
27
|
+
plugin = driver.instance
|
|
28
|
+
assert_equal plugin.class, Fluent::Plugin::ProtobufHttpInput
|
|
29
|
+
assert_equal plugin.bind, '0.0.0.0'
|
|
30
|
+
assert_equal plugin.port, 8080
|
|
31
|
+
assert_equal plugin.in_mode, :binary
|
|
32
|
+
assert_equal plugin.out_mode, :binary
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
sub_test_case 'route#emit' do
|
|
37
|
+
conf = %(
|
|
38
|
+
proto_dir ./test/data/protos
|
|
39
|
+
tag test
|
|
40
|
+
in_mode binary
|
|
41
|
+
out_mode json
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
test 'test invalid msgtype in query string i.e. empty or mismatch' do
|
|
45
|
+
driver = create_driver(conf)
|
|
46
|
+
res_codes = []
|
|
47
|
+
driver.run do
|
|
48
|
+
path = '/test'
|
|
49
|
+
res = post(path, @log_bin)
|
|
50
|
+
res_codes << res.code
|
|
51
|
+
end
|
|
52
|
+
assert_equal 1, res_codes.size
|
|
53
|
+
assert_equal '400', res_codes[0]
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
test 'test incoming type mismatch [in_mode != Content-Type]' do
|
|
57
|
+
conf = %(
|
|
58
|
+
proto_dir ./test/data/protos
|
|
59
|
+
tag test
|
|
60
|
+
in_mode binary
|
|
61
|
+
out_mode json
|
|
62
|
+
)
|
|
63
|
+
driver = create_driver(conf)
|
|
64
|
+
res_codes = []
|
|
65
|
+
driver.run do
|
|
66
|
+
path = '/test?msgtype=service.logging.Log'
|
|
67
|
+
res = post(path, @log_bin, :json)
|
|
68
|
+
res_codes << res.code
|
|
69
|
+
end
|
|
70
|
+
assert_equal 1, res_codes.size
|
|
71
|
+
assert_equal '400', res_codes[0]
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
test 'test single message (Binary to JSON)' do
|
|
75
|
+
driver = create_driver(conf)
|
|
76
|
+
res_codes = []
|
|
77
|
+
driver.run do
|
|
78
|
+
path = '/test?msgtype=service.logging.Log'
|
|
79
|
+
res = post(path, @log_bin)
|
|
80
|
+
res_codes << res.code
|
|
81
|
+
end
|
|
82
|
+
assert_equal 1, res_codes.size
|
|
83
|
+
assert_equal '200', res_codes[0]
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
test 'test single message (JSON to Binary)' do
|
|
87
|
+
conf = %(
|
|
88
|
+
proto_dir ./test/data/protos
|
|
89
|
+
tag test
|
|
90
|
+
in_mode json
|
|
91
|
+
out_mode binary
|
|
92
|
+
)
|
|
93
|
+
driver = create_driver(conf)
|
|
94
|
+
res_codes = []
|
|
95
|
+
driver.run do
|
|
96
|
+
path = '/test?msgtype=service.logging.Log'
|
|
97
|
+
res = post(path, @log_json, :json)
|
|
98
|
+
res_codes << res.code
|
|
99
|
+
end
|
|
100
|
+
assert_equal 1, res_codes.size
|
|
101
|
+
assert_equal '200', res_codes[0]
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
test 'test batch messages (Binary to JSON)' do
|
|
105
|
+
conf = %(
|
|
106
|
+
proto_dir ./test/data/protos
|
|
107
|
+
tag test
|
|
108
|
+
in_mode binary
|
|
109
|
+
out_mode json
|
|
110
|
+
)
|
|
111
|
+
driver = create_driver(conf)
|
|
112
|
+
res_codes = []
|
|
113
|
+
driver.run do
|
|
114
|
+
path = '/test?msgtype=service.logging.Batch&batch=true'
|
|
115
|
+
res = post(path, @log_bin_batch)
|
|
116
|
+
res_codes << res.code
|
|
117
|
+
end
|
|
118
|
+
assert_equal 1, res_codes.size
|
|
119
|
+
assert_equal '200', res_codes[0]
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
test 'test incompatible message' do
|
|
123
|
+
conf = %(
|
|
124
|
+
proto_dir ./test/data/protos
|
|
125
|
+
tag test
|
|
126
|
+
in_mode binary
|
|
127
|
+
out_mode json
|
|
128
|
+
)
|
|
129
|
+
driver = create_driver(conf)
|
|
130
|
+
res_codes = []
|
|
131
|
+
driver.run do
|
|
132
|
+
path = '/test?msgtype=service.logging.Log'
|
|
133
|
+
res = post(path, @log_bin_batch)
|
|
134
|
+
res_codes << res.code
|
|
135
|
+
end
|
|
136
|
+
assert_equal 1, res_codes.size
|
|
137
|
+
assert_equal '400', res_codes[0]
|
|
138
|
+
end
|
|
11
139
|
end
|
|
12
140
|
|
|
13
141
|
private
|
|
14
142
|
|
|
15
|
-
def
|
|
16
|
-
|
|
143
|
+
def post(path, body, type = :binary)
|
|
144
|
+
http = Net::HTTP.new('127.0.0.1', 8080)
|
|
145
|
+
content_type = 'application/octet-stream'
|
|
146
|
+
content_type = 'application/json' if type == :json
|
|
147
|
+
header = { 'Content-Type' => content_type }
|
|
148
|
+
req = Net::HTTP::Post.new(path, header)
|
|
149
|
+
req.body = body
|
|
150
|
+
http.request(req)
|
|
17
151
|
end
|
|
18
152
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-protobuf-http
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Azeem Sajid
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2021-05-15 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -16,14 +16,20 @@ dependencies:
|
|
|
16
16
|
requirements:
|
|
17
17
|
- - "~>"
|
|
18
18
|
- !ruby/object:Gem::Version
|
|
19
|
-
version: '1
|
|
19
|
+
version: '2.1'
|
|
20
|
+
- - ">="
|
|
21
|
+
- !ruby/object:Gem::Version
|
|
22
|
+
version: 2.1.0
|
|
20
23
|
type: :development
|
|
21
24
|
prerelease: false
|
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
26
|
requirements:
|
|
24
27
|
- - "~>"
|
|
25
28
|
- !ruby/object:Gem::Version
|
|
26
|
-
version: '1
|
|
29
|
+
version: '2.1'
|
|
30
|
+
- - ">="
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: 2.1.0
|
|
27
33
|
- !ruby/object:Gem::Dependency
|
|
28
34
|
name: rake
|
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -38,6 +44,26 @@ dependencies:
|
|
|
38
44
|
- - "~>"
|
|
39
45
|
- !ruby/object:Gem::Version
|
|
40
46
|
version: '12.0'
|
|
47
|
+
- !ruby/object:Gem::Dependency
|
|
48
|
+
name: simplecov
|
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
|
50
|
+
requirements:
|
|
51
|
+
- - "~>"
|
|
52
|
+
- !ruby/object:Gem::Version
|
|
53
|
+
version: '0.12'
|
|
54
|
+
- - "<="
|
|
55
|
+
- !ruby/object:Gem::Version
|
|
56
|
+
version: 0.12.2
|
|
57
|
+
type: :development
|
|
58
|
+
prerelease: false
|
|
59
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
60
|
+
requirements:
|
|
61
|
+
- - "~>"
|
|
62
|
+
- !ruby/object:Gem::Version
|
|
63
|
+
version: '0.12'
|
|
64
|
+
- - "<="
|
|
65
|
+
- !ruby/object:Gem::Version
|
|
66
|
+
version: 0.12.2
|
|
41
67
|
- !ruby/object:Gem::Dependency
|
|
42
68
|
name: test-unit
|
|
43
69
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -100,13 +126,19 @@ executables: []
|
|
|
100
126
|
extensions: []
|
|
101
127
|
extra_rdoc_files: []
|
|
102
128
|
files:
|
|
129
|
+
- ".github/workflows/ci.yml"
|
|
103
130
|
- ".gitignore"
|
|
131
|
+
- ".rubocop.yml"
|
|
104
132
|
- Gemfile
|
|
105
133
|
- LICENSE
|
|
106
134
|
- README.md
|
|
107
135
|
- Rakefile
|
|
108
136
|
- fluent-plugin-protobuf-http.gemspec
|
|
109
137
|
- lib/fluent/plugin/in_protobuf_http.rb
|
|
138
|
+
- test/data/log.bin
|
|
139
|
+
- test/data/log.json
|
|
140
|
+
- test/data/logbatch5.bin
|
|
141
|
+
- test/data/protos/log.proto
|
|
110
142
|
- test/helper.rb
|
|
111
143
|
- test/plugin/test_in_protobuf_http.rb
|
|
112
144
|
homepage: https://github.com/iamAzeem/fluent-plugin-protobuf-http
|
|
@@ -134,5 +166,9 @@ signing_key:
|
|
|
134
166
|
specification_version: 4
|
|
135
167
|
summary: fluentd HTTP Input Plugin for Protocol Buffers
|
|
136
168
|
test_files:
|
|
169
|
+
- test/data/log.bin
|
|
170
|
+
- test/data/log.json
|
|
171
|
+
- test/data/logbatch5.bin
|
|
172
|
+
- test/data/protos/log.proto
|
|
137
173
|
- test/helper.rb
|
|
138
174
|
- test/plugin/test_in_protobuf_http.rb
|