fluent-plugin-protobuf-http 0.3.0 → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/dependabot.yml +6 -0
- data/.github/workflows/ci.yml +15 -26
- data/README.md +68 -39
- data/fluent-plugin-protobuf-http.gemspec +2 -1
- data/lib/fluent/plugin/in_protobuf_http.rb +4 -3
- metadata +10 -10
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a559c0013395560b44bd1a1e2ca12898e481b932b8860f1cf31f557544031458
|
|
4
|
+
data.tar.gz: 827f41577acb0bbf3f526047030a662a3069eaeafe5fa3a648e1027dfb41f2fc
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 627665ee19fc340181a6c4e00912aa537b9b2f12bfff6c96c582180453302a6c88c756c583c9c7f003a19c57f9f48d799b912155dae8898e0a0bf54997030d9f
|
|
7
|
+
data.tar.gz: 6b85fda4bb44831b70023728cc8b10071d5a771f753f0c4054fa87a1f9ce650635dfd2a1fa9997c7e13ea798b65df8e874fd247de4d0339af20d7ebac6e93d02
|
data/.github/workflows/ci.yml
CHANGED
|
@@ -2,47 +2,36 @@ name: ci
|
|
|
2
2
|
|
|
3
3
|
on:
|
|
4
4
|
push:
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
- '.rubocop.yml'
|
|
5
|
+
branches: [main]
|
|
6
|
+
paths-ignore: ['**.md', '.rubocop.yml']
|
|
8
7
|
pull_request:
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
8
|
+
branches: [main]
|
|
9
|
+
types: [opened, synchronize, reopened]
|
|
10
|
+
paths-ignore: ['**.md', '.rubocop.yml']
|
|
12
11
|
|
|
13
12
|
jobs:
|
|
14
|
-
|
|
15
|
-
name: Run tests
|
|
13
|
+
build-and-test:
|
|
16
14
|
strategy:
|
|
17
15
|
matrix:
|
|
18
|
-
os: [ubuntu-
|
|
19
|
-
ruby-version: ['2.
|
|
16
|
+
os: [ubuntu-latest, macos-latest, windows-latest]
|
|
17
|
+
ruby-version: ['2.6', '2.7']
|
|
20
18
|
|
|
21
19
|
runs-on: ${{ matrix.os }}
|
|
22
20
|
|
|
23
21
|
steps:
|
|
24
|
-
-
|
|
22
|
+
- name: Checkout [${{ github.repository }}]
|
|
23
|
+
uses: actions/checkout@v4
|
|
25
24
|
|
|
26
|
-
- name: Set up Ruby
|
|
25
|
+
- name: Set up Ruby ${{ matrix.ruby-version }}
|
|
27
26
|
uses: ruby/setup-ruby@v1
|
|
28
27
|
with:
|
|
29
28
|
ruby-version: ${{ matrix.ruby-version }}
|
|
30
29
|
bundler-cache: true
|
|
31
30
|
|
|
32
|
-
- name: Bundle install
|
|
33
|
-
run: |
|
|
34
|
-
gem install bundler && \
|
|
35
|
-
bundle install --jobs 4 --retry 3
|
|
36
|
-
|
|
37
31
|
- name: Install protoc
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
unzip protoc-3.17.0-linux-x86_64.zip
|
|
42
|
-
chmod +x ./bin/protoc && sudo mv ./bin/protoc /usr/local/bin/protoc
|
|
43
|
-
sudo mv ./include/* /usr/local/include/
|
|
44
|
-
cd .. && rm -rf ./protoc
|
|
45
|
-
protoc --version
|
|
32
|
+
uses: arduino/setup-protoc@master
|
|
33
|
+
with:
|
|
34
|
+
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
|
46
35
|
|
|
47
|
-
- name:
|
|
36
|
+
- name: Run tests
|
|
48
37
|
run: bundle exec rake test
|
data/README.md
CHANGED
|
@@ -1,24 +1,24 @@
|
|
|
1
1
|
# fluent-plugin-protobuf-http
|
|
2
2
|
|
|
3
3
|
[](https://github.com/iamazeem/fluent-plugin-protobuf-http/actions/workflows/ci.yml)
|
|
4
|
-
[](https://github.com/iamAzeem/fluent-plugin-protobuf-http/blob/master/LICENSE)
|
|
5
5
|
|
|
6
|
-
[](https://rubygems.org/gems/fluent-plugin-protobuf-http)
|
|
7
|
+
[](https://www.buymeacoffee.com/iamazeem)
|
|
7
8
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
9
|
+
## Overview
|
|
11
10
|
|
|
12
|
-
[Fluentd](https://fluentd.org/) HTTP input plugin for
|
|
11
|
+
[Fluentd](https://fluentd.org/) HTTP input plugin for
|
|
12
|
+
[Protocol Buffers](https://github.com/protocolbuffers/protobuf).
|
|
13
13
|
|
|
14
14
|
## Features
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
|
|
16
|
+
- Automatic compilation of `.proto` files located in `proto_dir`
|
|
17
|
+
- Incoming Format: Binary or JSON (`Content-Type`: `application/octet-stream` or
|
|
18
18
|
`application/json`)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
19
|
+
- Outgoing Format: Binary or JSON
|
|
20
|
+
- Single and Batch message support
|
|
21
|
+
- TLS Support with `<transport>` section and `https://` URL protocol prefix.
|
|
22
22
|
|
|
23
23
|
For more details on TLS configuration, see this official
|
|
24
24
|
[example](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-server#configuration-example).
|
|
@@ -33,7 +33,7 @@ gem install fluent-plugin-protobuf-http
|
|
|
33
33
|
|
|
34
34
|
### Bundler
|
|
35
35
|
|
|
36
|
-
Add following line to your Gemfile:
|
|
36
|
+
Add the following line to your Gemfile:
|
|
37
37
|
|
|
38
38
|
```ruby
|
|
39
39
|
gem 'fluent-plugin-protobuf-http'
|
|
@@ -47,25 +47,25 @@ bundle
|
|
|
47
47
|
|
|
48
48
|
## Configuration
|
|
49
49
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
50
|
+
- `bind` (string) (optional): The address to listen to.
|
|
51
|
+
- Default: `0.0.0.0`
|
|
52
|
+
- `port` (integer) (optional): The port to listen to.
|
|
53
|
+
- Default: `8080`
|
|
54
|
+
- `proto_dir` (string) (required): The directory path that contains the .proto files.
|
|
55
|
+
- `in_mode` (enum) (optional): The mode of incoming (supported) events.
|
|
56
|
+
- Modes: `binary`, `json`
|
|
57
|
+
- Default: `binary`
|
|
58
|
+
- `out_mode` (enum) (optional): The mode of outgoing (emitted) events.
|
|
59
|
+
- Modes: `binary`, `json`
|
|
60
|
+
- Default: `binary`
|
|
61
|
+
- `tag` (string) (required): The tag for the event.
|
|
62
62
|
|
|
63
63
|
### `<transport>` section (optional) (single)
|
|
64
64
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
65
|
+
- `protocol` (enum) (optional):
|
|
66
|
+
- Protocols: `tcp`, `tls`
|
|
67
|
+
- Default: `tcp`
|
|
68
|
+
- For more details, see this official configuration
|
|
69
69
|
[example](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-server#configuration-example).
|
|
70
70
|
|
|
71
71
|
### Example
|
|
@@ -150,8 +150,8 @@ message Batch {
|
|
|
150
150
|
}
|
|
151
151
|
```
|
|
152
152
|
|
|
153
|
-
IMPORTANT
|
|
154
|
-
file! You can choose any name for a batch message type.
|
|
153
|
+
**IMPORTANT**: The `Batch` message type is part of `log.proto`, it is not a
|
|
154
|
+
separate file! You can choose any name for a batch message type.
|
|
155
155
|
|
|
156
156
|
Here is the complete `log.proto` file:
|
|
157
157
|
|
|
@@ -263,9 +263,9 @@ For a simple use-case of incoming HTTP events and their routing to
|
|
|
263
263
|
</match>
|
|
264
264
|
```
|
|
265
265
|
|
|
266
|
-
The incoming binary messages will be
|
|
266
|
+
The incoming binary messages will be converted to JSON for further consumption.
|
|
267
267
|
|
|
268
|
-
### Single Message
|
|
268
|
+
### Single Message Use-case
|
|
269
269
|
|
|
270
270
|
Test Parameters:
|
|
271
271
|
|
|
@@ -297,10 +297,10 @@ curl -X POST -H "Content-Type: application/octet-stream" \
|
|
|
297
297
|
2020-06-09 18:53:47 +0500 [info]: #0 [protobuf_http_input] [S] {json} [127.0.0.1:41222, msgtype: service.logging.Log, size: 183 bytes]
|
|
298
298
|
```
|
|
299
299
|
|
|
300
|
-
For sample Single
|
|
301
|
-
|
|
300
|
+
For generating sample Single messages, see
|
|
301
|
+
https://github.com/iamAzeem/protobuf-log-sample.
|
|
302
302
|
|
|
303
|
-
### Batch Message
|
|
303
|
+
### Batch Message Use-case
|
|
304
304
|
|
|
305
305
|
Test Parameters:
|
|
306
306
|
|
|
@@ -363,16 +363,45 @@ $ curl -X POST -H "Content-Type: application/octet-stream" \
|
|
|
363
363
|
```
|
|
364
364
|
|
|
365
365
|
For sample Batch message generation, see
|
|
366
|
-
[this](https://gist.github.com/iamAzeem/a8a24092132e1741a76956192f2104cc).
|
|
366
|
+
[this gist](https://gist.github.com/iamAzeem/a8a24092132e1741a76956192f2104cc).
|
|
367
|
+
|
|
368
|
+
## CI Workflow
|
|
369
|
+
|
|
370
|
+
The [CI workflow](ci..github/workflows/ci.yml) sets up the prerequisites. It
|
|
371
|
+
builds and installs the plugin, and then runs the automated tests.
|
|
372
|
+
|
|
373
|
+
To run tests locally, run:
|
|
374
|
+
|
|
375
|
+
```shell
|
|
376
|
+
bundle exec rake test
|
|
377
|
+
```
|
|
378
|
+
|
|
379
|
+
The [test](./test) directory contains the tests and the [input](./test/data)
|
|
380
|
+
files.
|
|
381
|
+
|
|
382
|
+
The code coverage is printed at the end using `simplecov`.
|
|
383
|
+
|
|
384
|
+
## Known Issues
|
|
385
|
+
|
|
386
|
+
- This plugin internally uses the HTTP server plugin
|
|
387
|
+
[helper](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-http_server)
|
|
388
|
+
which has higher precedence for `async-http` over `webrick`. But,
|
|
389
|
+
[`webrick`](https://github.com/ruby/webrick) is required to run this. In an
|
|
390
|
+
environment where both are installed, `async-http` is automatically selected
|
|
391
|
+
causing runtime issues. To make this work, you need to uninstall `async-http`
|
|
392
|
+
i.e. `gem uninstall async-http`. See issue
|
|
393
|
+
[#10](https://github.com/iamazeem/fluent-plugin-protobuf-http/issues/10) for
|
|
394
|
+
more details where this was identified in Docker containers where both gems
|
|
395
|
+
were already installed.
|
|
367
396
|
|
|
368
397
|
## Contribute
|
|
369
398
|
|
|
370
|
-
- Fork the project.
|
|
399
|
+
- [Fork](https://github.com/iamazeem/fluent-plugin-protobuf-http/fork) the project.
|
|
371
400
|
- Check out the latest `main` branch.
|
|
372
|
-
- Create a feature or bugfix branch from `main`.
|
|
401
|
+
- Create a `feature` or `bugfix` branch from `main`.
|
|
373
402
|
- Commit and push your changes.
|
|
374
403
|
- Make sure to add and run tests locally: `bundle exec rake test`.
|
|
375
|
-
- Run
|
|
404
|
+
- Run [Rubocop](https://github.com/rubocop/rubocop) and fix the lint errors.
|
|
376
405
|
- Submit the PR.
|
|
377
406
|
|
|
378
407
|
## License
|
|
@@ -5,7 +5,8 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
5
5
|
|
|
6
6
|
Gem::Specification.new do |spec|
|
|
7
7
|
spec.name = 'fluent-plugin-protobuf-http'
|
|
8
|
-
spec.version = '0.3.
|
|
8
|
+
spec.version = '0.3.1'
|
|
9
|
+
spec.date = '2023-09-16'
|
|
9
10
|
spec.authors = ['Azeem Sajid']
|
|
10
11
|
spec.email = ['azeem.sajid@gmail.com']
|
|
11
12
|
|
|
@@ -116,9 +116,10 @@ module Fluent
|
|
|
116
116
|
log.debug("Extracting message types [#{compiled_proto}]...")
|
|
117
117
|
msg_types = []
|
|
118
118
|
File.foreach(compiled_proto) do |line|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
119
|
+
line.strip!
|
|
120
|
+
if line.include?('::Google::Protobuf::DescriptorPool.generated_pool.lookup') && line.end_with?('.msgclass')
|
|
121
|
+
extracted_msg_type = line[/"([^"]*)"/, 1].freeze
|
|
122
|
+
msg_types.push(extracted_msg_type) unless extracted_msg_type.nil?
|
|
122
123
|
end
|
|
123
124
|
end
|
|
124
125
|
|
metadata
CHANGED
|
@@ -1,35 +1,35 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-protobuf-http
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Azeem Sajid
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2023-09-16 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
|
16
16
|
requirements:
|
|
17
|
-
- - "~>"
|
|
18
|
-
- !ruby/object:Gem::Version
|
|
19
|
-
version: '2.1'
|
|
20
17
|
- - ">="
|
|
21
18
|
- !ruby/object:Gem::Version
|
|
22
19
|
version: 2.1.0
|
|
20
|
+
- - "~>"
|
|
21
|
+
- !ruby/object:Gem::Version
|
|
22
|
+
version: '2.1'
|
|
23
23
|
type: :development
|
|
24
24
|
prerelease: false
|
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
|
26
26
|
requirements:
|
|
27
|
-
- - "~>"
|
|
28
|
-
- !ruby/object:Gem::Version
|
|
29
|
-
version: '2.1'
|
|
30
27
|
- - ">="
|
|
31
28
|
- !ruby/object:Gem::Version
|
|
32
29
|
version: 2.1.0
|
|
30
|
+
- - "~>"
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: '2.1'
|
|
33
33
|
- !ruby/object:Gem::Dependency
|
|
34
34
|
name: rake
|
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -126,6 +126,7 @@ executables: []
|
|
|
126
126
|
extensions: []
|
|
127
127
|
extra_rdoc_files: []
|
|
128
128
|
files:
|
|
129
|
+
- ".github/dependabot.yml"
|
|
129
130
|
- ".github/workflows/ci.yml"
|
|
130
131
|
- ".gitignore"
|
|
131
132
|
- ".rubocop.yml"
|
|
@@ -160,8 +161,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
160
161
|
- !ruby/object:Gem::Version
|
|
161
162
|
version: '0'
|
|
162
163
|
requirements: []
|
|
163
|
-
|
|
164
|
-
rubygems_version: 2.7.6
|
|
164
|
+
rubygems_version: 3.0.9
|
|
165
165
|
signing_key:
|
|
166
166
|
specification_version: 4
|
|
167
167
|
summary: fluentd HTTP Input Plugin for Protocol Buffers
|