fluent-plugin-protobuf-http 0.2.0 → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/dependabot.yml +6 -0
- data/.github/workflows/ci.yml +37 -0
- data/.rubocop.yml +19 -0
- data/Gemfile +2 -0
- data/README.md +202 -94
- data/Rakefile +2 -0
- data/fluent-plugin-protobuf-http.gemspec +6 -2
- data/lib/fluent/plugin/in_protobuf_http.rb +24 -25
- data/test/data/log.bin +3 -0
- data/test/data/log.json +1 -0
- data/test/data/logbatch5.bin +12 -0
- data/test/data/protos/log.proto +31 -0
- data/test/helper.rb +10 -5
- data/test/plugin/test_in_protobuf_http.rb +140 -6
- metadata +42 -6
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a559c0013395560b44bd1a1e2ca12898e481b932b8860f1cf31f557544031458
|
|
4
|
+
data.tar.gz: 827f41577acb0bbf3f526047030a662a3069eaeafe5fa3a648e1027dfb41f2fc
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 627665ee19fc340181a6c4e00912aa537b9b2f12bfff6c96c582180453302a6c88c756c583c9c7f003a19c57f9f48d799b912155dae8898e0a0bf54997030d9f
|
|
7
|
+
data.tar.gz: 6b85fda4bb44831b70023728cc8b10071d5a771f753f0c4054fa87a1f9ce650635dfd2a1fa9997c7e13ea798b65df8e874fd247de4d0339af20d7ebac6e93d02
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
paths-ignore: ['**.md', '.rubocop.yml']
|
|
7
|
+
pull_request:
|
|
8
|
+
branches: [main]
|
|
9
|
+
types: [opened, synchronize, reopened]
|
|
10
|
+
paths-ignore: ['**.md', '.rubocop.yml']
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
build-and-test:
|
|
14
|
+
strategy:
|
|
15
|
+
matrix:
|
|
16
|
+
os: [ubuntu-latest, macos-latest, windows-latest]
|
|
17
|
+
ruby-version: ['2.6', '2.7']
|
|
18
|
+
|
|
19
|
+
runs-on: ${{ matrix.os }}
|
|
20
|
+
|
|
21
|
+
steps:
|
|
22
|
+
- name: Checkout [${{ github.repository }}]
|
|
23
|
+
uses: actions/checkout@v4
|
|
24
|
+
|
|
25
|
+
- name: Set up Ruby ${{ matrix.ruby-version }}
|
|
26
|
+
uses: ruby/setup-ruby@v1
|
|
27
|
+
with:
|
|
28
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
29
|
+
bundler-cache: true
|
|
30
|
+
|
|
31
|
+
- name: Install protoc
|
|
32
|
+
uses: arduino/setup-protoc@master
|
|
33
|
+
with:
|
|
34
|
+
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
|
35
|
+
|
|
36
|
+
- name: Run tests
|
|
37
|
+
run: bundle exec rake test
|
data/.rubocop.yml
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# RuboCop Default Config: https://github.com/rubocop-hq/rubocop/blob/master/config/default.yml
|
|
2
|
+
|
|
3
|
+
AllCops:
|
|
4
|
+
NewCops: disable
|
|
5
|
+
|
|
6
|
+
Gemspec/RequiredRubyVersion:
|
|
7
|
+
Enabled: false
|
|
8
|
+
|
|
9
|
+
Metrics/ClassLength:
|
|
10
|
+
Enabled: false
|
|
11
|
+
|
|
12
|
+
Metrics/MethodLength:
|
|
13
|
+
Enabled: false
|
|
14
|
+
|
|
15
|
+
Metrics/AbcSize:
|
|
16
|
+
Enabled: false
|
|
17
|
+
|
|
18
|
+
Layout/LineLength:
|
|
19
|
+
Enabled: false
|
data/Gemfile
CHANGED
data/README.md
CHANGED
|
@@ -1,62 +1,79 @@
|
|
|
1
1
|
# fluent-plugin-protobuf-http
|
|
2
2
|
|
|
3
|
-
[
|
|
3
|
+
[](https://github.com/iamazeem/fluent-plugin-protobuf-http/actions/workflows/ci.yml)
|
|
4
|
+
[](https://github.com/iamAzeem/fluent-plugin-protobuf-http/blob/master/LICENSE)
|
|
5
|
+
|
|
6
|
+
[](https://rubygems.org/gems/fluent-plugin-protobuf-http)
|
|
7
|
+
[](https://www.buymeacoffee.com/iamazeem)
|
|
8
|
+
|
|
9
|
+
## Overview
|
|
10
|
+
|
|
11
|
+
[Fluentd](https://fluentd.org/) HTTP input plugin for
|
|
12
|
+
[Protocol Buffers](https://github.com/protocolbuffers/protobuf).
|
|
4
13
|
|
|
5
14
|
## Features
|
|
6
15
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
16
|
+
- Automatic compilation of `.proto` files located in `proto_dir`
|
|
17
|
+
- Incoming Format: Binary or JSON (`Content-Type`: `application/octet-stream` or
|
|
18
|
+
`application/json`)
|
|
19
|
+
- Outgoing Format: Binary or JSON
|
|
20
|
+
- Single and Batch message support
|
|
21
|
+
- TLS Support with `<transport>` section and `https://` URL protocol prefix.
|
|
22
|
+
|
|
23
|
+
For more details on TLS configuration, see this official
|
|
24
|
+
[example](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-server#configuration-example).
|
|
12
25
|
|
|
13
26
|
## Installation
|
|
14
27
|
|
|
15
28
|
### RubyGems
|
|
16
29
|
|
|
17
|
-
```
|
|
18
|
-
|
|
30
|
+
```shell
|
|
31
|
+
gem install fluent-plugin-protobuf-http
|
|
19
32
|
```
|
|
20
33
|
|
|
21
34
|
### Bundler
|
|
22
35
|
|
|
23
|
-
Add following line to your Gemfile:
|
|
36
|
+
Add the following line to your Gemfile:
|
|
37
|
+
|
|
24
38
|
```ruby
|
|
25
|
-
gem
|
|
39
|
+
gem 'fluent-plugin-protobuf-http'
|
|
26
40
|
```
|
|
27
41
|
|
|
28
42
|
And then execute:
|
|
29
|
-
|
|
30
|
-
|
|
43
|
+
|
|
44
|
+
```shell
|
|
45
|
+
bundle
|
|
31
46
|
```
|
|
32
47
|
|
|
33
48
|
## Configuration
|
|
34
49
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
###
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
50
|
+
- `bind` (string) (optional): The address to listen to.
|
|
51
|
+
- Default: `0.0.0.0`
|
|
52
|
+
- `port` (integer) (optional): The port to listen to.
|
|
53
|
+
- Default: `8080`
|
|
54
|
+
- `proto_dir` (string) (required): The directory path that contains the .proto files.
|
|
55
|
+
- `in_mode` (enum) (optional): The mode of incoming (supported) events.
|
|
56
|
+
- Modes: `binary`, `json`
|
|
57
|
+
- Default: `binary`
|
|
58
|
+
- `out_mode` (enum) (optional): The mode of outgoing (emitted) events.
|
|
59
|
+
- Modes: `binary`, `json`
|
|
60
|
+
- Default: `binary`
|
|
61
|
+
- `tag` (string) (required): The tag for the event.
|
|
62
|
+
|
|
63
|
+
### `<transport>` section (optional) (single)
|
|
64
|
+
|
|
65
|
+
- `protocol` (enum) (optional):
|
|
66
|
+
- Protocols: `tcp`, `tls`
|
|
67
|
+
- Default: `tcp`
|
|
68
|
+
- For more details, see this official configuration
|
|
69
|
+
[example](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-server#configuration-example).
|
|
54
70
|
|
|
55
71
|
### Example
|
|
56
72
|
|
|
57
|
-
```
|
|
58
|
-
#
|
|
59
|
-
#
|
|
73
|
+
```text
|
|
74
|
+
# Endpoints:
|
|
75
|
+
# - Single Message: http://ip:port/<tag>?msgtype=<msgtype>
|
|
76
|
+
# - Batch Message: http://ip:port/<tag>?msgtype=<batch-msgtype>?batch=true
|
|
60
77
|
|
|
61
78
|
<source>
|
|
62
79
|
@type protobuf_http
|
|
@@ -74,12 +91,18 @@ $ bundle
|
|
|
74
91
|
|
|
75
92
|
## Schemas (`.proto` files)
|
|
76
93
|
|
|
77
|
-
The
|
|
78
|
-
|
|
79
|
-
|
|
94
|
+
The prime use-case for this plugin is assumed to be event logging. So, always
|
|
95
|
+
use self-contained `.proto` file(s) that do not import other `.proto` files. The
|
|
96
|
+
names e.g. `package`, `message`, etc. must be unique and are treated as
|
|
97
|
+
case-sensitive.
|
|
80
98
|
|
|
81
|
-
Consider this
|
|
82
|
-
|
|
99
|
+
Consider this
|
|
100
|
+
[log.proto](https://github.com/iamAzeem/protobuf-log-sample/blob/master/log.proto)
|
|
101
|
+
schema from
|
|
102
|
+
[protobuf-log-sample](https://github.com/iamAzeem/protobuf-log-sample)
|
|
103
|
+
repository:
|
|
104
|
+
|
|
105
|
+
```protobuf
|
|
83
106
|
syntax = "proto3";
|
|
84
107
|
|
|
85
108
|
package service.logging;
|
|
@@ -108,9 +131,9 @@ message Log {
|
|
|
108
131
|
}
|
|
109
132
|
```
|
|
110
133
|
|
|
111
|
-
The fully-qualified message type for `Log`
|
|
112
|
-
|
|
113
|
-
|
|
134
|
+
The fully-qualified message type for `Log` is `service.logging.Log`. This
|
|
135
|
+
message type is used as the value of `msgtype` query parameter in the URL. See
|
|
136
|
+
URL section below for more on `msgtype`.
|
|
114
137
|
|
|
115
138
|
### Single Message
|
|
116
139
|
|
|
@@ -118,20 +141,21 @@ The above schema will be used as-is for the single message.
|
|
|
118
141
|
|
|
119
142
|
### Batch Message
|
|
120
143
|
|
|
121
|
-
For
|
|
122
|
-
|
|
144
|
+
For the batch message, the schema must be like this:
|
|
145
|
+
|
|
146
|
+
```protobuf
|
|
123
147
|
message Batch {
|
|
124
148
|
string type = 1;
|
|
125
149
|
repeated Log batch = 2;
|
|
126
150
|
}
|
|
127
151
|
```
|
|
128
152
|
|
|
129
|
-
IMPORTANT
|
|
130
|
-
|
|
131
|
-
You can choose any name for a batch message type.
|
|
153
|
+
**IMPORTANT**: The `Batch` message type is part of `log.proto`, it is not a
|
|
154
|
+
separate file! You can choose any name for a batch message type.
|
|
132
155
|
|
|
133
|
-
|
|
134
|
-
|
|
156
|
+
Here is the complete `log.proto` file:
|
|
157
|
+
|
|
158
|
+
```protobuf
|
|
135
159
|
syntax = "proto3";
|
|
136
160
|
|
|
137
161
|
package service.logging;
|
|
@@ -165,50 +189,62 @@ message Batch {
|
|
|
165
189
|
}
|
|
166
190
|
```
|
|
167
191
|
|
|
168
|
-
For batch processing, the plugin looks for special members `type` and `batch`.
|
|
192
|
+
For batch processing, the plugin looks for special members `type` and `batch`.
|
|
169
193
|
The `type` will indicate the message type of `batch` i.e. `Log` in this example.
|
|
170
194
|
|
|
171
|
-
The type of `Batch` is `service.logging.Batch` and it will be the value of
|
|
172
|
-
The type of `batch` array is `service.logging.Log`
|
|
195
|
+
The type of `Batch` is `service.logging.Batch` and it will be the value of
|
|
196
|
+
`msgtype` in the URL query. The type of `batch` array is `service.logging.Log`
|
|
197
|
+
and it will be the value of `type`.
|
|
173
198
|
|
|
174
|
-
The `google.protobuf.Any` type has not been used deliberately
|
|
175
|
-
|
|
176
|
-
|
|
199
|
+
The `google.protobuf.Any` type has not been used here deliberately. It stores
|
|
200
|
+
the message type with each message resulting in an increase in size. Refer to
|
|
201
|
+
[protobuf-repeated-type-vs-any](https://github.com/iamAzeem/protobuf-repeated-type-vs-any)
|
|
202
|
+
for a simple comparison. With the above approach, the type is stored only once
|
|
203
|
+
for the whole batch.
|
|
177
204
|
|
|
178
205
|
### Endpoint (URL)
|
|
179
206
|
|
|
180
207
|
For single message:
|
|
181
|
-
|
|
208
|
+
|
|
209
|
+
```text
|
|
182
210
|
http://<ip>:<port>/<tag>?msgtype=<fully-qualified-message-type>
|
|
183
211
|
```
|
|
184
212
|
|
|
185
213
|
For batch message:
|
|
186
|
-
|
|
214
|
+
|
|
215
|
+
```text
|
|
187
216
|
http://<ip>:<port>/<tag>?msgtype=<fully-qualified-message-type-for-batch>&batch=true
|
|
188
217
|
```
|
|
189
218
|
|
|
190
|
-
Without `batch=true` query parameter, the batch will be treated as a single
|
|
219
|
+
Without `batch=true` query parameter, the batch will be treated as a single
|
|
220
|
+
message.
|
|
191
221
|
|
|
192
|
-
For example, for a log type `service.logging.Log` and its corresponding batch
|
|
222
|
+
For example, for a log type `service.logging.Log` and its corresponding batch
|
|
223
|
+
type `service.logging.Batch`, the URLs would be:
|
|
193
224
|
|
|
194
|
-
|
|
195
|
-
|
|
225
|
+
For single message:
|
|
226
|
+
|
|
227
|
+
```text
|
|
196
228
|
http://localhost:8080/debug.test?msgtype=service.logging.Log
|
|
197
229
|
```
|
|
198
230
|
|
|
199
|
-
|
|
200
|
-
|
|
231
|
+
For batch message:
|
|
232
|
+
|
|
233
|
+
```text
|
|
201
234
|
http://localhost:8080/debug.test?msgtype=service.logging.Batch&batch=true
|
|
202
235
|
```
|
|
203
236
|
|
|
204
|
-
**NOTE**: The values of query parameters (`msgtype`, `batch`) are
|
|
237
|
+
**NOTE**: The values of query parameters (`msgtype`, `batch`) are
|
|
238
|
+
case-sensitive!
|
|
205
239
|
|
|
206
240
|
## Test Use-Case (`curl`)
|
|
207
241
|
|
|
208
|
-
For a simple
|
|
242
|
+
For a simple use-case of incoming HTTP events and their routing to
|
|
243
|
+
[stdout](https://docs.fluentd.org/output/stdout) may be configured like this:
|
|
209
244
|
|
|
210
245
|
`fluent.conf`:
|
|
211
|
-
|
|
246
|
+
|
|
247
|
+
```text
|
|
212
248
|
<source>
|
|
213
249
|
@type protobuf_http
|
|
214
250
|
@id protobuf_http_input
|
|
@@ -224,24 +260,36 @@ For a simple test use-case of events and their routing to [stdout](https://docs.
|
|
|
224
260
|
|
|
225
261
|
<match debug.test>
|
|
226
262
|
@type stdout
|
|
227
|
-
@id stdout_output
|
|
228
263
|
</match>
|
|
229
264
|
```
|
|
230
265
|
|
|
231
|
-
The incoming binary messages will be
|
|
266
|
+
The incoming binary messages will be converted to JSON for further consumption.
|
|
232
267
|
|
|
233
|
-
|
|
268
|
+
### Single Message Use-case
|
|
234
269
|
|
|
235
|
-
Test
|
|
236
|
-
* data: `log.bin`, msgtype: `service.logging.Log`
|
|
270
|
+
Test Parameters:
|
|
237
271
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
272
|
+
| input file | single message type |
|
|
273
|
+
|:----------:|:---------------------:|
|
|
274
|
+
| `log.bin` | `service.logging.Log` |
|
|
275
|
+
|
|
276
|
+
URL:
|
|
277
|
+
|
|
278
|
+
```bash
|
|
279
|
+
http://localhost:8080/debug.test?msgtype=service.logging.Log
|
|
241
280
|
```
|
|
242
281
|
|
|
243
|
-
`
|
|
282
|
+
`curl` command:
|
|
283
|
+
|
|
284
|
+
```shell
|
|
285
|
+
curl -X POST -H "Content-Type: application/octet-stream" \
|
|
286
|
+
--data-binary "@/<path>/log.bin" \
|
|
287
|
+
"http://localhost:8080/debug.test?msgtype=service.logging.Log"
|
|
244
288
|
```
|
|
289
|
+
|
|
290
|
+
`fluentd` logs (Observe JSON at the end):
|
|
291
|
+
|
|
292
|
+
```text
|
|
245
293
|
2020-06-09 18:53:47 +0500 [info]: #0 [protobuf_http_input] [R] {binary} [127.0.0.1:41222, size: 86 bytes]
|
|
246
294
|
2020-06-09 18:53:47 +0500 [warn]: #0 [protobuf_http_input] 'batch' not found in 'query_string' [msgtype=service.logging.Log]
|
|
247
295
|
2020-06-09 18:53:47 +0500 [info]: #0 [protobuf_http_input] [S] {binary} [127.0.0.1:41222, msgtype: service.logging.Log, size: 86 bytes]
|
|
@@ -249,22 +297,38 @@ $ curl -X POST -H "Content-Type: application/octet-stream" --data-binary "@/<pat
|
|
|
249
297
|
2020-06-09 18:53:47 +0500 [info]: #0 [protobuf_http_input] [S] {json} [127.0.0.1:41222, msgtype: service.logging.Log, size: 183 bytes]
|
|
250
298
|
```
|
|
251
299
|
|
|
252
|
-
For
|
|
300
|
+
For generating sample Single messages, see
|
|
301
|
+
https://github.com/iamAzeem/protobuf-log-sample.
|
|
253
302
|
|
|
254
|
-
|
|
303
|
+
### Batch Message Use-case
|
|
255
304
|
|
|
256
|
-
Test
|
|
257
|
-
* data: `logbatch2.bin`, msgtype: `service.logging.Batch`, type: `service.logging.Log` [batch_size: 2 messages]
|
|
258
|
-
* data: `logbatch5.bin`, msgtype: `service.logging.Batch`, type: `service.logging.Log` [batch_size: 5 messages]
|
|
305
|
+
Test Parameters:
|
|
259
306
|
|
|
260
|
-
|
|
307
|
+
| input file | batch message type | batch internal type | messages |
|
|
308
|
+
|:---------------:|:-----------------------:|:---------------------:|:--------:|
|
|
309
|
+
| `logbatch2.bin` | `service.logging.Batch` | `service.logging.Log` | 2 |
|
|
310
|
+
| `logbatch5.bin` | `service.logging.Batch` | `service.logging.Log` | 5 |
|
|
311
|
+
|
|
312
|
+
URL:
|
|
313
|
+
|
|
314
|
+
```text
|
|
315
|
+
http://localhost:8080/debug.test?msgtype=service.logging.Batch&batch=true
|
|
261
316
|
```
|
|
262
|
-
|
|
317
|
+
|
|
318
|
+
**`logbatch2.bin`**
|
|
319
|
+
|
|
320
|
+
`curl` command:
|
|
321
|
+
|
|
322
|
+
```shell
|
|
323
|
+
$ curl -X POST -H "Content-Type: application/octet-stream" \
|
|
324
|
+
--data-binary "@/<path>/logbatch2.bin" \
|
|
325
|
+
"http://localhost:8080/debug.test?msgtype=service.logging.Batch&batch=true"
|
|
263
326
|
{"status":"Batch received! [batch_type: service.logging.Log, batch_size: 2 messages]"}
|
|
264
327
|
```
|
|
265
328
|
|
|
266
|
-
`fluentd`
|
|
267
|
-
|
|
329
|
+
`fluentd` logs:
|
|
330
|
+
|
|
331
|
+
```text
|
|
268
332
|
2020-06-09 19:04:13 +0500 [info]: #0 [protobuf_http_input] [R] {binary} [127.0.0.1:41416, size: 207 bytes]
|
|
269
333
|
2020-06-09 19:04:13 +0500 [info]: #0 [protobuf_http_input] [B] {binary} [127.0.0.1:41416, msgtype: service.logging.Batch, size: 207 bytes]
|
|
270
334
|
2020-06-09 19:04:13 +0500 [info]: #0 [protobuf_http_input] [B] Emitting message stream/batch [batch_size: 2 messages]...
|
|
@@ -273,14 +337,20 @@ $ curl -X POST -H "Content-Type: application/octet-stream" --data-binary "@/<pat
|
|
|
273
337
|
2020-06-09 19:04:13 +0500 [info]: #0 [protobuf_http_input] [B] {json} [127.0.0.1:41416, msgtype: service.logging.Batch] Batch received! [batch_type: service.logging.Log, batch_size: 2 messages]
|
|
274
338
|
```
|
|
275
339
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
340
|
+
**`logbatch5.bin`**
|
|
341
|
+
|
|
342
|
+
`curl` command:
|
|
343
|
+
|
|
344
|
+
```bash
|
|
345
|
+
$ curl -X POST -H "Content-Type: application/octet-stream" \
|
|
346
|
+
--data-binary "@/<path>/logbatch5.bin" \
|
|
347
|
+
"http://localhost:8080/debug.test?msgtype=service.logging.Batch&batch=true"
|
|
279
348
|
{"status":"Batch received! [batch_type: service.logging.Log, batch_size: 5 messages]"}
|
|
280
349
|
```
|
|
281
350
|
|
|
282
|
-
`fluentd`
|
|
283
|
-
|
|
351
|
+
`fluentd` logs:
|
|
352
|
+
|
|
353
|
+
```text
|
|
284
354
|
2020-06-09 19:07:09 +0500 [info]: #0 [protobuf_http_input] [R] {binary} [127.0.0.1:41552, size: 486 bytes]
|
|
285
355
|
2020-06-09 19:07:09 +0500 [info]: #0 [protobuf_http_input] [B] {binary} [127.0.0.1:41552, msgtype: service.logging.Batch, size: 486 bytes]
|
|
286
356
|
2020-06-09 19:07:09 +0500 [info]: #0 [protobuf_http_input] [B] Emitting message stream/batch [batch_size: 5 messages]...
|
|
@@ -292,10 +362,48 @@ $ curl -X POST -H "Content-Type: application/octet-stream" --data-binary "@/<pat
|
|
|
292
362
|
2020-06-09 19:07:09 +0500 [info]: #0 [protobuf_http_input] [B] {json} [127.0.0.1:41552, msgtype: service.logging.Batch] Batch received! [batch_type: service.logging.Log, batch_size: 5 messages]
|
|
293
363
|
```
|
|
294
364
|
|
|
295
|
-
For
|
|
365
|
+
For sample Batch message generation, see
|
|
366
|
+
[this gist](https://gist.github.com/iamAzeem/a8a24092132e1741a76956192f2104cc).
|
|
367
|
+
|
|
368
|
+
## CI Workflow
|
|
369
|
+
|
|
370
|
+
The [CI workflow](ci..github/workflows/ci.yml) sets up the prerequisites. It
|
|
371
|
+
builds and installs the plugin, and then runs the automated tests.
|
|
372
|
+
|
|
373
|
+
To run tests locally, run:
|
|
374
|
+
|
|
375
|
+
```shell
|
|
376
|
+
bundle exec rake test
|
|
377
|
+
```
|
|
378
|
+
|
|
379
|
+
The [test](./test) directory contains the tests and the [input](./test/data)
|
|
380
|
+
files.
|
|
381
|
+
|
|
382
|
+
The code coverage is printed at the end using `simplecov`.
|
|
383
|
+
|
|
384
|
+
## Known Issues
|
|
385
|
+
|
|
386
|
+
- This plugin internally uses the HTTP server plugin
|
|
387
|
+
[helper](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-http_server)
|
|
388
|
+
which has higher precedence for `async-http` over `webrick`. But,
|
|
389
|
+
[`webrick`](https://github.com/ruby/webrick) is required to run this. In an
|
|
390
|
+
environment where both are installed, `async-http` is automatically selected
|
|
391
|
+
causing runtime issues. To make this work, you need to uninstall `async-http`
|
|
392
|
+
i.e. `gem uninstall async-http`. See issue
|
|
393
|
+
[#10](https://github.com/iamazeem/fluent-plugin-protobuf-http/issues/10) for
|
|
394
|
+
more details where this was identified in Docker containers where both gems
|
|
395
|
+
were already installed.
|
|
396
|
+
|
|
397
|
+
## Contribute
|
|
398
|
+
|
|
399
|
+
- [Fork](https://github.com/iamazeem/fluent-plugin-protobuf-http/fork) the project.
|
|
400
|
+
- Check out the latest `main` branch.
|
|
401
|
+
- Create a `feature` or `bugfix` branch from `main`.
|
|
402
|
+
- Commit and push your changes.
|
|
403
|
+
- Make sure to add and run tests locally: `bundle exec rake test`.
|
|
404
|
+
- Run [Rubocop](https://github.com/rubocop/rubocop) and fix the lint errors.
|
|
405
|
+
- Submit the PR.
|
|
296
406
|
|
|
297
|
-
##
|
|
407
|
+
## License
|
|
298
408
|
|
|
299
|
-
|
|
300
|
-
* License
|
|
301
|
-
* Apache License, Version 2.0
|
|
409
|
+
[Apache 2.0](LICENSE)
|
data/Rakefile
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
|
+
# frozen-string-literal: true
|
|
2
|
+
|
|
1
3
|
lib = File.expand_path('../lib', __dir__)
|
|
2
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
3
5
|
|
|
4
6
|
Gem::Specification.new do |spec|
|
|
5
7
|
spec.name = 'fluent-plugin-protobuf-http'
|
|
6
|
-
spec.version = '0.
|
|
8
|
+
spec.version = '0.3.1'
|
|
9
|
+
spec.date = '2023-09-16'
|
|
7
10
|
spec.authors = ['Azeem Sajid']
|
|
8
11
|
spec.email = ['azeem.sajid@gmail.com']
|
|
9
12
|
|
|
@@ -20,8 +23,9 @@ Gem::Specification.new do |spec|
|
|
|
20
23
|
spec.test_files = test_files
|
|
21
24
|
spec.require_paths = ['lib']
|
|
22
25
|
|
|
23
|
-
spec.add_development_dependency 'bundler', '~> 1.
|
|
26
|
+
spec.add_development_dependency 'bundler', '~> 2.1', '>= 2.1.0'
|
|
24
27
|
spec.add_development_dependency 'rake', '~> 12.0'
|
|
28
|
+
spec.add_development_dependency 'simplecov', '~> 0.12', '<= 0.12.2'
|
|
25
29
|
spec.add_development_dependency 'test-unit', '~> 3.0'
|
|
26
30
|
spec.add_runtime_dependency 'fluentd', ['>= 0.14.10', '< 2']
|
|
27
31
|
spec.add_runtime_dependency 'google-protobuf', '~> 3.12', '>= 3.12.2'
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen-string-literal: true
|
|
2
|
+
|
|
1
3
|
#
|
|
2
4
|
# Copyright 2020 Azeem Sajid
|
|
3
5
|
#
|
|
@@ -18,9 +20,11 @@ require 'fluent/config/error'
|
|
|
18
20
|
require 'fluent/plugin_helper/http_server'
|
|
19
21
|
require 'webrick/httputils'
|
|
20
22
|
require 'json'
|
|
23
|
+
require 'English'
|
|
21
24
|
|
|
22
25
|
module Fluent
|
|
23
26
|
module Plugin
|
|
27
|
+
# Implementation of HTTP input plugin for Protobuf
|
|
24
28
|
class ProtobufHttpInput < Fluent::Plugin::Input
|
|
25
29
|
Fluent::Plugin.register_input('protobuf_http', self)
|
|
26
30
|
|
|
@@ -46,10 +50,6 @@ module Fluent
|
|
|
46
50
|
config_argument :protocol, :enum, list: %i[tcp tls], default: :tcp
|
|
47
51
|
end
|
|
48
52
|
|
|
49
|
-
config_section :transform, required: false, multi: false, init: true, param_name: :transform_config do
|
|
50
|
-
config_argument :msgtype, :string
|
|
51
|
-
end
|
|
52
|
-
|
|
53
53
|
def initialize
|
|
54
54
|
super
|
|
55
55
|
|
|
@@ -61,16 +61,16 @@ module Fluent
|
|
|
61
61
|
def compile_protos
|
|
62
62
|
log.debug("Checking proto_dir [#{@proto_dir}]...")
|
|
63
63
|
|
|
64
|
-
path = File.expand_path(@proto_dir)
|
|
64
|
+
path = File.expand_path(@proto_dir)
|
|
65
65
|
raise Fluent::ConfigError, "protos_dir does not exist! [#{path}]" unless Dir.exist?(path)
|
|
66
66
|
|
|
67
|
-
@protos = Dir["#{path}/*.proto"]
|
|
67
|
+
@protos = Dir["#{path}/*.proto"]
|
|
68
68
|
raise Fluent::ConfigError, "Empty proto_dir! [#{path}]" unless @protos.any?
|
|
69
69
|
|
|
70
70
|
log.info("Compiling .proto files [#{@protos.length}]...")
|
|
71
71
|
|
|
72
72
|
`protoc --ruby_out=#{path} --proto_path=#{path} #{path}/*.proto`
|
|
73
|
-
raise Fluent::ConfigError, 'Could not compile! See error(s) above.' unless
|
|
73
|
+
raise Fluent::ConfigError, 'Could not compile! See error(s) above.' unless $CHILD_STATUS.success?
|
|
74
74
|
|
|
75
75
|
log.info("Compiled successfully:\n- #{@protos.join("\n- ")}")
|
|
76
76
|
|
|
@@ -82,8 +82,8 @@ module Fluent
|
|
|
82
82
|
end
|
|
83
83
|
|
|
84
84
|
def get_compiled_proto(proto)
|
|
85
|
-
proto_suffix = '.proto'
|
|
86
|
-
compiled_proto_suffix = '_pb.rb'
|
|
85
|
+
proto_suffix = '.proto'
|
|
86
|
+
compiled_proto_suffix = '_pb.rb'
|
|
87
87
|
|
|
88
88
|
compiled_proto = proto.chomp(proto_suffix) + compiled_proto_suffix
|
|
89
89
|
raise Fluent::ConfigError, "Compiled proto not found! [#{compiled_proto}]" unless File.file?(compiled_proto)
|
|
@@ -116,9 +116,10 @@ module Fluent
|
|
|
116
116
|
log.debug("Extracting message types [#{compiled_proto}]...")
|
|
117
117
|
msg_types = []
|
|
118
118
|
File.foreach(compiled_proto) do |line|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
119
|
+
line.strip!
|
|
120
|
+
if line.include?('::Google::Protobuf::DescriptorPool.generated_pool.lookup') && line.end_with?('.msgclass')
|
|
121
|
+
extracted_msg_type = line[/"([^"]*)"/, 1].freeze
|
|
122
|
+
msg_types.push(extracted_msg_type) unless extracted_msg_type.nil?
|
|
122
123
|
end
|
|
123
124
|
end
|
|
124
125
|
|
|
@@ -152,15 +153,13 @@ module Fluent
|
|
|
152
153
|
tls_opts = @transport_config.to_h
|
|
153
154
|
end
|
|
154
155
|
|
|
155
|
-
log.warn("#{@transform_config.to_h}")
|
|
156
|
-
|
|
157
156
|
log.info("Starting protobuf #{proto == :tcp ? 'HTTP' : 'HTTPS'} server [#{@bind}:#{@port}]...")
|
|
158
157
|
log.debug("TLS configuration:\n#{tls_opts}") if tls_opts
|
|
159
158
|
|
|
160
159
|
http_server_create_http_server(:protobuf_server, addr: @bind, port: @port, logger: log, proto: proto, tls_opts: tls_opts) do |server|
|
|
161
160
|
server.post("/#{tag}") do |req|
|
|
162
|
-
peeraddr = "#{req.peeraddr[2]}:#{req.peeraddr[1]}"
|
|
163
|
-
serialized_msg = req.body
|
|
161
|
+
peeraddr = "#{req.peeraddr[2]}:#{req.peeraddr[1]}" # ip:port
|
|
162
|
+
serialized_msg = req.body
|
|
164
163
|
|
|
165
164
|
log.info("[R] {#{@in_mode}} [#{peeraddr}, size: #{serialized_msg.length} bytes]")
|
|
166
165
|
log.debug("Dumping serialized message [#{serialized_msg.length} bytes]:\n#{serialized_msg}")
|
|
@@ -168,7 +167,7 @@ module Fluent
|
|
|
168
167
|
content_type = req.header['content-type'][0]
|
|
169
168
|
|
|
170
169
|
unless valid_content_type?(content_type)
|
|
171
|
-
status = "Invalid 'Content-Type' header! [#{content_type}]"
|
|
170
|
+
status = "Invalid 'Content-Type' header! [#{content_type}]"
|
|
172
171
|
log.warn("[X] Message rejected! [#{peeraddr}] #{status}")
|
|
173
172
|
next [400, { 'Content-Type' => 'application/json', 'Connection' => 'close' }, { 'status' => status }.to_json]
|
|
174
173
|
end
|
|
@@ -177,7 +176,7 @@ module Fluent
|
|
|
177
176
|
|
|
178
177
|
msgtype, batch = get_query_params(req.query_string)
|
|
179
178
|
unless @msgclass_lookup.key?(msgtype)
|
|
180
|
-
status = "Invalid 'msgtype' in 'query_string'! [#{msgtype}]"
|
|
179
|
+
status = "Invalid 'msgtype' in 'query_string'! [#{msgtype}]"
|
|
181
180
|
log.warn("[X] Message rejected! [#{peeraddr}] #{status}")
|
|
182
181
|
next [400, { 'Content-Type' => 'application/json', 'Connection' => 'close' }, { 'status' => status }.to_json]
|
|
183
182
|
end
|
|
@@ -187,7 +186,7 @@ module Fluent
|
|
|
187
186
|
deserialized_msg = deserialize_msg(msgtype, serialized_msg)
|
|
188
187
|
|
|
189
188
|
if deserialized_msg.nil?
|
|
190
|
-
status = "Incompatible message! [msgtype: #{msgtype}, size: #{serialized_msg.length} bytes]"
|
|
189
|
+
status = "Incompatible message! [msgtype: #{msgtype}, size: #{serialized_msg.length} bytes]"
|
|
191
190
|
log.warn("[X] Message rejected! [#{peeraddr}] #{status}")
|
|
192
191
|
next [400, { 'Content-Type' => 'application/json', 'Connection' => 'close' }, { 'status' => status }.to_json]
|
|
193
192
|
end
|
|
@@ -214,7 +213,7 @@ module Fluent
|
|
|
214
213
|
log.info("[B] {#{@in_mode}} [#{peeraddr}, msgtype: #{msgtype}, size: #{serialized_msg.length} bytes]")
|
|
215
214
|
|
|
216
215
|
if deserialized_msg.type.nil? || deserialized_msg.batch.nil? || deserialized_msg.batch.empty?
|
|
217
|
-
status = "Invalid 'batch' message! [msgtype: #{msgtype}, size: #{serialized_msg.length} bytes]"
|
|
216
|
+
status = "Invalid 'batch' message! [msgtype: #{msgtype}, size: #{serialized_msg.length} bytes]"
|
|
218
217
|
log.warn("[X] Message rejected! [#{peeraddr}] #{status}")
|
|
219
218
|
next [400, { 'Content-Type' => 'application/json', 'Connection' => 'close' }, { 'status' => status }.to_json]
|
|
220
219
|
end
|
|
@@ -234,7 +233,7 @@ module Fluent
|
|
|
234
233
|
|
|
235
234
|
router.emit_stream(@tag, stream)
|
|
236
235
|
|
|
237
|
-
status = "Batch received! [batch_type: #{batch_type}, batch_size: #{batch_size} messages]"
|
|
236
|
+
status = "Batch received! [batch_type: #{batch_type}, batch_size: #{batch_size} messages]"
|
|
238
237
|
log.info("[B] {#{@out_mode}} [#{peeraddr}, msgtype: #{msgtype}] #{status}")
|
|
239
238
|
[200, { 'Content-Type' => 'application/json', 'Connection' => 'close' }, { 'status' => status }.to_json]
|
|
240
239
|
end
|
|
@@ -242,8 +241,8 @@ module Fluent
|
|
|
242
241
|
end
|
|
243
242
|
|
|
244
243
|
def valid_content_type?(content_type)
|
|
245
|
-
hdr_binary = 'application/octet-stream'
|
|
246
|
-
hdr_json = 'application/json'
|
|
244
|
+
hdr_binary = 'application/octet-stream'
|
|
245
|
+
hdr_json = 'application/json'
|
|
247
246
|
|
|
248
247
|
case @in_mode
|
|
249
248
|
when :binary
|
|
@@ -284,7 +283,7 @@ module Fluent
|
|
|
284
283
|
rescue Google::Protobuf::ParseError => e
|
|
285
284
|
log.error("Incompatible message! [msgtype: #{msgtype}, size: #{serialized_msg.length} bytes] #{e}")
|
|
286
285
|
nil
|
|
287
|
-
rescue => e
|
|
286
|
+
rescue StandardError => e
|
|
288
287
|
log.error("Deserializaton failed! Error: #{e}")
|
|
289
288
|
nil
|
|
290
289
|
end
|
|
@@ -300,7 +299,7 @@ module Fluent
|
|
|
300
299
|
when :json
|
|
301
300
|
msgclass.encode_json(deserialized_msg)
|
|
302
301
|
end
|
|
303
|
-
rescue => e
|
|
302
|
+
rescue StandardError => e
|
|
304
303
|
log.error("Serialization failed! [msgtype: #{msgtype}, msg: #{deserialized_msg}] Error: #{e}")
|
|
305
304
|
nil
|
|
306
305
|
end
|
data/test/data/log.bin
ADDED
data/test/data/log.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"context":{"timestamp":"2020-06-01T16:24:19Z","hostOrIp":"192.168.xxx.xxx","serviceName":"test","user":"test"},"level":"INFO","message":"This is a test log generated by [./log.rb]."}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
|
|
2
|
+
service.logging.Log[
|
|
3
|
+
%
|
|
4
|
+
����192.168.xxx.xxxtest"test0This is a test log generated by [./logbatch.rb].[
|
|
5
|
+
%
|
|
6
|
+
����192.168.xxx.xxxtest"test0This is a test log generated by [./logbatch.rb].[
|
|
7
|
+
%
|
|
8
|
+
����192.168.xxx.xxxtest"test0This is a test log generated by [./logbatch.rb].[
|
|
9
|
+
%
|
|
10
|
+
����192.168.xxx.xxxtest"test0This is a test log generated by [./logbatch.rb].[
|
|
11
|
+
%
|
|
12
|
+
����192.168.xxx.xxxtest"test0This is a test log generated by [./logbatch.rb].
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
syntax = "proto3";
|
|
2
|
+
|
|
3
|
+
package service.logging;
|
|
4
|
+
|
|
5
|
+
import "google/protobuf/timestamp.proto";
|
|
6
|
+
|
|
7
|
+
message Log {
|
|
8
|
+
message Context {
|
|
9
|
+
google.protobuf.Timestamp timestamp = 1;
|
|
10
|
+
string host_or_ip = 2;
|
|
11
|
+
string service_name = 3;
|
|
12
|
+
string user = 4;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
enum Level {
|
|
16
|
+
DEBUG = 0;
|
|
17
|
+
INFO = 1;
|
|
18
|
+
WARN = 2;
|
|
19
|
+
ERROR = 3;
|
|
20
|
+
FATAL = 4;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
Context context = 1;
|
|
24
|
+
Level level = 2;
|
|
25
|
+
string message = 3;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
message Batch {
|
|
29
|
+
string type = 1;
|
|
30
|
+
repeated Log batch = 2;
|
|
31
|
+
}
|
data/test/helper.rb
CHANGED
|
@@ -1,8 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
require
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
# frozen-string-literal: true
|
|
2
|
+
|
|
3
|
+
require 'simplecov'
|
|
4
|
+
SimpleCov.start
|
|
5
|
+
|
|
6
|
+
$LOAD_PATH.unshift(File.expand_path('..', __dir__))
|
|
7
|
+
require 'test-unit'
|
|
8
|
+
require 'fluent/test'
|
|
9
|
+
require 'fluent/test/driver/input'
|
|
10
|
+
require 'fluent/test/helpers'
|
|
6
11
|
|
|
7
12
|
Test::Unit::TestCase.include(Fluent::Test::Helpers)
|
|
8
13
|
Test::Unit::TestCase.extend(Fluent::Test::Helpers)
|
|
@@ -1,18 +1,152 @@
|
|
|
1
|
-
|
|
2
|
-
require "fluent/plugin/in_protobuf_http.rb"
|
|
1
|
+
# frozen-string-literal: true
|
|
3
2
|
|
|
3
|
+
require 'helper'
|
|
4
|
+
require 'fluent/plugin/in_protobuf_http'
|
|
5
|
+
require 'net/http'
|
|
6
|
+
|
|
7
|
+
# Implementation of Test Class
|
|
4
8
|
class ProtobufHttpInputTest < Test::Unit::TestCase
|
|
5
9
|
setup do
|
|
6
10
|
Fluent::Test.setup
|
|
11
|
+
@log_bin = File.open('./test/data/log.bin', 'rb') { |f| f.read }
|
|
12
|
+
@log_json = File.open('./test/data/log.json', 'r') { |f| f.read }
|
|
13
|
+
@log_bin_batch = File.open('./test/data/logbatch5.bin', 'rb') { |f| f.read }
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def create_driver(conf)
|
|
17
|
+
Fluent::Test::Driver::Input.new(Fluent::Plugin::ProtobufHttpInput).configure(conf)
|
|
7
18
|
end
|
|
8
19
|
|
|
9
|
-
|
|
10
|
-
|
|
20
|
+
sub_test_case 'configure' do
|
|
21
|
+
test 'test default configuration' do
|
|
22
|
+
conf = %(
|
|
23
|
+
proto_dir ./test/data/protos
|
|
24
|
+
tag test
|
|
25
|
+
)
|
|
26
|
+
driver = create_driver(conf)
|
|
27
|
+
plugin = driver.instance
|
|
28
|
+
assert_equal plugin.class, Fluent::Plugin::ProtobufHttpInput
|
|
29
|
+
assert_equal plugin.bind, '0.0.0.0'
|
|
30
|
+
assert_equal plugin.port, 8080
|
|
31
|
+
assert_equal plugin.in_mode, :binary
|
|
32
|
+
assert_equal plugin.out_mode, :binary
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
sub_test_case 'route#emit' do
|
|
37
|
+
conf = %(
|
|
38
|
+
proto_dir ./test/data/protos
|
|
39
|
+
tag test
|
|
40
|
+
in_mode binary
|
|
41
|
+
out_mode json
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
test 'test invalid msgtype in query string i.e. empty or mismatch' do
|
|
45
|
+
driver = create_driver(conf)
|
|
46
|
+
res_codes = []
|
|
47
|
+
driver.run do
|
|
48
|
+
path = '/test'
|
|
49
|
+
res = post(path, @log_bin)
|
|
50
|
+
res_codes << res.code
|
|
51
|
+
end
|
|
52
|
+
assert_equal 1, res_codes.size
|
|
53
|
+
assert_equal '400', res_codes[0]
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
test 'test incoming type mismatch [in_mode != Content-Type]' do
|
|
57
|
+
conf = %(
|
|
58
|
+
proto_dir ./test/data/protos
|
|
59
|
+
tag test
|
|
60
|
+
in_mode binary
|
|
61
|
+
out_mode json
|
|
62
|
+
)
|
|
63
|
+
driver = create_driver(conf)
|
|
64
|
+
res_codes = []
|
|
65
|
+
driver.run do
|
|
66
|
+
path = '/test?msgtype=service.logging.Log'
|
|
67
|
+
res = post(path, @log_bin, :json)
|
|
68
|
+
res_codes << res.code
|
|
69
|
+
end
|
|
70
|
+
assert_equal 1, res_codes.size
|
|
71
|
+
assert_equal '400', res_codes[0]
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
test 'test single message (Binary to JSON)' do
|
|
75
|
+
driver = create_driver(conf)
|
|
76
|
+
res_codes = []
|
|
77
|
+
driver.run do
|
|
78
|
+
path = '/test?msgtype=service.logging.Log'
|
|
79
|
+
res = post(path, @log_bin)
|
|
80
|
+
res_codes << res.code
|
|
81
|
+
end
|
|
82
|
+
assert_equal 1, res_codes.size
|
|
83
|
+
assert_equal '200', res_codes[0]
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
test 'test single message (JSON to Binary)' do
|
|
87
|
+
conf = %(
|
|
88
|
+
proto_dir ./test/data/protos
|
|
89
|
+
tag test
|
|
90
|
+
in_mode json
|
|
91
|
+
out_mode binary
|
|
92
|
+
)
|
|
93
|
+
driver = create_driver(conf)
|
|
94
|
+
res_codes = []
|
|
95
|
+
driver.run do
|
|
96
|
+
path = '/test?msgtype=service.logging.Log'
|
|
97
|
+
res = post(path, @log_json, :json)
|
|
98
|
+
res_codes << res.code
|
|
99
|
+
end
|
|
100
|
+
assert_equal 1, res_codes.size
|
|
101
|
+
assert_equal '200', res_codes[0]
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
test 'test batch messages (Binary to JSON)' do
|
|
105
|
+
conf = %(
|
|
106
|
+
proto_dir ./test/data/protos
|
|
107
|
+
tag test
|
|
108
|
+
in_mode binary
|
|
109
|
+
out_mode json
|
|
110
|
+
)
|
|
111
|
+
driver = create_driver(conf)
|
|
112
|
+
res_codes = []
|
|
113
|
+
driver.run do
|
|
114
|
+
path = '/test?msgtype=service.logging.Batch&batch=true'
|
|
115
|
+
res = post(path, @log_bin_batch)
|
|
116
|
+
res_codes << res.code
|
|
117
|
+
end
|
|
118
|
+
assert_equal 1, res_codes.size
|
|
119
|
+
assert_equal '200', res_codes[0]
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
test 'test incompatible message' do
|
|
123
|
+
conf = %(
|
|
124
|
+
proto_dir ./test/data/protos
|
|
125
|
+
tag test
|
|
126
|
+
in_mode binary
|
|
127
|
+
out_mode json
|
|
128
|
+
)
|
|
129
|
+
driver = create_driver(conf)
|
|
130
|
+
res_codes = []
|
|
131
|
+
driver.run do
|
|
132
|
+
path = '/test?msgtype=service.logging.Log'
|
|
133
|
+
res = post(path, @log_bin_batch)
|
|
134
|
+
res_codes << res.code
|
|
135
|
+
end
|
|
136
|
+
assert_equal 1, res_codes.size
|
|
137
|
+
assert_equal '400', res_codes[0]
|
|
138
|
+
end
|
|
11
139
|
end
|
|
12
140
|
|
|
13
141
|
private
|
|
14
142
|
|
|
15
|
-
def
|
|
16
|
-
|
|
143
|
+
def post(path, body, type = :binary)
|
|
144
|
+
http = Net::HTTP.new('127.0.0.1', 8080)
|
|
145
|
+
content_type = 'application/octet-stream'
|
|
146
|
+
content_type = 'application/json' if type == :json
|
|
147
|
+
header = { 'Content-Type' => content_type }
|
|
148
|
+
req = Net::HTTP::Post.new(path, header)
|
|
149
|
+
req.body = body
|
|
150
|
+
http.request(req)
|
|
17
151
|
end
|
|
18
152
|
end
|
metadata
CHANGED
|
@@ -1,29 +1,35 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-protobuf-http
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Azeem Sajid
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2023-09-16 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
|
16
16
|
requirements:
|
|
17
|
+
- - ">="
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: 2.1.0
|
|
17
20
|
- - "~>"
|
|
18
21
|
- !ruby/object:Gem::Version
|
|
19
|
-
version: '1
|
|
22
|
+
version: '2.1'
|
|
20
23
|
type: :development
|
|
21
24
|
prerelease: false
|
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
26
|
requirements:
|
|
27
|
+
- - ">="
|
|
28
|
+
- !ruby/object:Gem::Version
|
|
29
|
+
version: 2.1.0
|
|
24
30
|
- - "~>"
|
|
25
31
|
- !ruby/object:Gem::Version
|
|
26
|
-
version: '1
|
|
32
|
+
version: '2.1'
|
|
27
33
|
- !ruby/object:Gem::Dependency
|
|
28
34
|
name: rake
|
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -38,6 +44,26 @@ dependencies:
|
|
|
38
44
|
- - "~>"
|
|
39
45
|
- !ruby/object:Gem::Version
|
|
40
46
|
version: '12.0'
|
|
47
|
+
- !ruby/object:Gem::Dependency
|
|
48
|
+
name: simplecov
|
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
|
50
|
+
requirements:
|
|
51
|
+
- - "~>"
|
|
52
|
+
- !ruby/object:Gem::Version
|
|
53
|
+
version: '0.12'
|
|
54
|
+
- - "<="
|
|
55
|
+
- !ruby/object:Gem::Version
|
|
56
|
+
version: 0.12.2
|
|
57
|
+
type: :development
|
|
58
|
+
prerelease: false
|
|
59
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
60
|
+
requirements:
|
|
61
|
+
- - "~>"
|
|
62
|
+
- !ruby/object:Gem::Version
|
|
63
|
+
version: '0.12'
|
|
64
|
+
- - "<="
|
|
65
|
+
- !ruby/object:Gem::Version
|
|
66
|
+
version: 0.12.2
|
|
41
67
|
- !ruby/object:Gem::Dependency
|
|
42
68
|
name: test-unit
|
|
43
69
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -100,13 +126,20 @@ executables: []
|
|
|
100
126
|
extensions: []
|
|
101
127
|
extra_rdoc_files: []
|
|
102
128
|
files:
|
|
129
|
+
- ".github/dependabot.yml"
|
|
130
|
+
- ".github/workflows/ci.yml"
|
|
103
131
|
- ".gitignore"
|
|
132
|
+
- ".rubocop.yml"
|
|
104
133
|
- Gemfile
|
|
105
134
|
- LICENSE
|
|
106
135
|
- README.md
|
|
107
136
|
- Rakefile
|
|
108
137
|
- fluent-plugin-protobuf-http.gemspec
|
|
109
138
|
- lib/fluent/plugin/in_protobuf_http.rb
|
|
139
|
+
- test/data/log.bin
|
|
140
|
+
- test/data/log.json
|
|
141
|
+
- test/data/logbatch5.bin
|
|
142
|
+
- test/data/protos/log.proto
|
|
110
143
|
- test/helper.rb
|
|
111
144
|
- test/plugin/test_in_protobuf_http.rb
|
|
112
145
|
homepage: https://github.com/iamAzeem/fluent-plugin-protobuf-http
|
|
@@ -128,11 +161,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
128
161
|
- !ruby/object:Gem::Version
|
|
129
162
|
version: '0'
|
|
130
163
|
requirements: []
|
|
131
|
-
|
|
132
|
-
rubygems_version: 2.7.6
|
|
164
|
+
rubygems_version: 3.0.9
|
|
133
165
|
signing_key:
|
|
134
166
|
specification_version: 4
|
|
135
167
|
summary: fluentd HTTP Input Plugin for Protocol Buffers
|
|
136
168
|
test_files:
|
|
169
|
+
- test/data/log.bin
|
|
170
|
+
- test/data/log.json
|
|
171
|
+
- test/data/logbatch5.bin
|
|
172
|
+
- test/data/protos/log.proto
|
|
137
173
|
- test/helper.rb
|
|
138
174
|
- test/plugin/test_in_protobuf_http.rb
|