fluent-plugin-protobuf-http 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/dependabot.yml +6 -0
- data/.github/workflows/ci.yml +15 -26
- data/README.md +68 -39
- data/fluent-plugin-protobuf-http.gemspec +2 -1
- data/lib/fluent/plugin/in_protobuf_http.rb +4 -3
- metadata +10 -10
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a559c0013395560b44bd1a1e2ca12898e481b932b8860f1cf31f557544031458
|
|
4
|
+
data.tar.gz: 827f41577acb0bbf3f526047030a662a3069eaeafe5fa3a648e1027dfb41f2fc
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 627665ee19fc340181a6c4e00912aa537b9b2f12bfff6c96c582180453302a6c88c756c583c9c7f003a19c57f9f48d799b912155dae8898e0a0bf54997030d9f
|
|
7
|
+
data.tar.gz: 6b85fda4bb44831b70023728cc8b10071d5a771f753f0c4054fa87a1f9ce650635dfd2a1fa9997c7e13ea798b65df8e874fd247de4d0339af20d7ebac6e93d02
|
data/.github/workflows/ci.yml
CHANGED
|
@@ -2,47 +2,36 @@ name: ci
|
|
|
2
2
|
|
|
3
3
|
on:
|
|
4
4
|
push:
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
- '.rubocop.yml'
|
|
5
|
+
branches: [main]
|
|
6
|
+
paths-ignore: ['**.md', '.rubocop.yml']
|
|
8
7
|
pull_request:
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
8
|
+
branches: [main]
|
|
9
|
+
types: [opened, synchronize, reopened]
|
|
10
|
+
paths-ignore: ['**.md', '.rubocop.yml']
|
|
12
11
|
|
|
13
12
|
jobs:
|
|
14
|
-
|
|
15
|
-
name: Run tests
|
|
13
|
+
build-and-test:
|
|
16
14
|
strategy:
|
|
17
15
|
matrix:
|
|
18
|
-
os: [ubuntu-
|
|
19
|
-
ruby-version: ['2.
|
|
16
|
+
os: [ubuntu-latest, macos-latest, windows-latest]
|
|
17
|
+
ruby-version: ['2.6', '2.7']
|
|
20
18
|
|
|
21
19
|
runs-on: ${{ matrix.os }}
|
|
22
20
|
|
|
23
21
|
steps:
|
|
24
|
-
-
|
|
22
|
+
- name: Checkout [${{ github.repository }}]
|
|
23
|
+
uses: actions/checkout@v4
|
|
25
24
|
|
|
26
|
-
- name: Set up Ruby
|
|
25
|
+
- name: Set up Ruby ${{ matrix.ruby-version }}
|
|
27
26
|
uses: ruby/setup-ruby@v1
|
|
28
27
|
with:
|
|
29
28
|
ruby-version: ${{ matrix.ruby-version }}
|
|
30
29
|
bundler-cache: true
|
|
31
30
|
|
|
32
|
-
- name: Bundle install
|
|
33
|
-
run: |
|
|
34
|
-
gem install bundler && \
|
|
35
|
-
bundle install --jobs 4 --retry 3
|
|
36
|
-
|
|
37
31
|
- name: Install protoc
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
unzip protoc-3.17.0-linux-x86_64.zip
|
|
42
|
-
chmod +x ./bin/protoc && sudo mv ./bin/protoc /usr/local/bin/protoc
|
|
43
|
-
sudo mv ./include/* /usr/local/include/
|
|
44
|
-
cd .. && rm -rf ./protoc
|
|
45
|
-
protoc --version
|
|
32
|
+
uses: arduino/setup-protoc@master
|
|
33
|
+
with:
|
|
34
|
+
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
|
46
35
|
|
|
47
|
-
- name:
|
|
36
|
+
- name: Run tests
|
|
48
37
|
run: bundle exec rake test
|
data/README.md
CHANGED
|
@@ -1,24 +1,24 @@
|
|
|
1
1
|
# fluent-plugin-protobuf-http
|
|
2
2
|
|
|
3
3
|
[](https://github.com/iamazeem/fluent-plugin-protobuf-http/actions/workflows/ci.yml)
|
|
4
|
-
[](https://github.com/iamAzeem/fluent-plugin-protobuf-http/blob/master/LICENSE)
|
|
5
5
|
|
|
6
|
-
[](https://rubygems.org/gems/fluent-plugin-protobuf-http)
|
|
7
|
+
[](https://www.buymeacoffee.com/iamazeem)
|
|
7
8
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
9
|
+
## Overview
|
|
11
10
|
|
|
12
|
-
[Fluentd](https://fluentd.org/) HTTP input plugin for
|
|
11
|
+
[Fluentd](https://fluentd.org/) HTTP input plugin for
|
|
12
|
+
[Protocol Buffers](https://github.com/protocolbuffers/protobuf).
|
|
13
13
|
|
|
14
14
|
## Features
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
|
|
16
|
+
- Automatic compilation of `.proto` files located in `proto_dir`
|
|
17
|
+
- Incoming Format: Binary or JSON (`Content-Type`: `application/octet-stream` or
|
|
18
18
|
`application/json`)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
19
|
+
- Outgoing Format: Binary or JSON
|
|
20
|
+
- Single and Batch message support
|
|
21
|
+
- TLS Support with `<transport>` section and `https://` URL protocol prefix.
|
|
22
22
|
|
|
23
23
|
For more details on TLS configuration, see this official
|
|
24
24
|
[example](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-server#configuration-example).
|
|
@@ -33,7 +33,7 @@ gem install fluent-plugin-protobuf-http
|
|
|
33
33
|
|
|
34
34
|
### Bundler
|
|
35
35
|
|
|
36
|
-
Add following line to your Gemfile:
|
|
36
|
+
Add the following line to your Gemfile:
|
|
37
37
|
|
|
38
38
|
```ruby
|
|
39
39
|
gem 'fluent-plugin-protobuf-http'
|
|
@@ -47,25 +47,25 @@ bundle
|
|
|
47
47
|
|
|
48
48
|
## Configuration
|
|
49
49
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
50
|
+
- `bind` (string) (optional): The address to listen to.
|
|
51
|
+
- Default: `0.0.0.0`
|
|
52
|
+
- `port` (integer) (optional): The port to listen to.
|
|
53
|
+
- Default: `8080`
|
|
54
|
+
- `proto_dir` (string) (required): The directory path that contains the .proto files.
|
|
55
|
+
- `in_mode` (enum) (optional): The mode of incoming (supported) events.
|
|
56
|
+
- Modes: `binary`, `json`
|
|
57
|
+
- Default: `binary`
|
|
58
|
+
- `out_mode` (enum) (optional): The mode of outgoing (emitted) events.
|
|
59
|
+
- Modes: `binary`, `json`
|
|
60
|
+
- Default: `binary`
|
|
61
|
+
- `tag` (string) (required): The tag for the event.
|
|
62
62
|
|
|
63
63
|
### `<transport>` section (optional) (single)
|
|
64
64
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
65
|
+
- `protocol` (enum) (optional):
|
|
66
|
+
- Protocols: `tcp`, `tls`
|
|
67
|
+
- Default: `tcp`
|
|
68
|
+
- For more details, see this official configuration
|
|
69
69
|
[example](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-server#configuration-example).
|
|
70
70
|
|
|
71
71
|
### Example
|
|
@@ -150,8 +150,8 @@ message Batch {
|
|
|
150
150
|
}
|
|
151
151
|
```
|
|
152
152
|
|
|
153
|
-
IMPORTANT
|
|
154
|
-
file! You can choose any name for a batch message type.
|
|
153
|
+
**IMPORTANT**: The `Batch` message type is part of `log.proto`, it is not a
|
|
154
|
+
separate file! You can choose any name for a batch message type.
|
|
155
155
|
|
|
156
156
|
Here is the complete `log.proto` file:
|
|
157
157
|
|
|
@@ -263,9 +263,9 @@ For a simple use-case of incoming HTTP events and their routing to
|
|
|
263
263
|
</match>
|
|
264
264
|
```
|
|
265
265
|
|
|
266
|
-
The incoming binary messages will be
|
|
266
|
+
The incoming binary messages will be converted to JSON for further consumption.
|
|
267
267
|
|
|
268
|
-
### Single Message
|
|
268
|
+
### Single Message Use-case
|
|
269
269
|
|
|
270
270
|
Test Parameters:
|
|
271
271
|
|
|
@@ -297,10 +297,10 @@ curl -X POST -H "Content-Type: application/octet-stream" \
|
|
|
297
297
|
2020-06-09 18:53:47 +0500 [info]: #0 [protobuf_http_input] [S] {json} [127.0.0.1:41222, msgtype: service.logging.Log, size: 183 bytes]
|
|
298
298
|
```
|
|
299
299
|
|
|
300
|
-
For sample Single
|
|
301
|
-
|
|
300
|
+
For generating sample Single messages, see
|
|
301
|
+
https://github.com/iamAzeem/protobuf-log-sample.
|
|
302
302
|
|
|
303
|
-
### Batch Message
|
|
303
|
+
### Batch Message Use-case
|
|
304
304
|
|
|
305
305
|
Test Parameters:
|
|
306
306
|
|
|
@@ -363,16 +363,45 @@ $ curl -X POST -H "Content-Type: application/octet-stream" \
|
|
|
363
363
|
```
|
|
364
364
|
|
|
365
365
|
For sample Batch message generation, see
|
|
366
|
-
[this](https://gist.github.com/iamAzeem/a8a24092132e1741a76956192f2104cc).
|
|
366
|
+
[this gist](https://gist.github.com/iamAzeem/a8a24092132e1741a76956192f2104cc).
|
|
367
|
+
|
|
368
|
+
## CI Workflow
|
|
369
|
+
|
|
370
|
+
The [CI workflow](ci..github/workflows/ci.yml) sets up the prerequisites. It
|
|
371
|
+
builds and installs the plugin, and then runs the automated tests.
|
|
372
|
+
|
|
373
|
+
To run tests locally, run:
|
|
374
|
+
|
|
375
|
+
```shell
|
|
376
|
+
bundle exec rake test
|
|
377
|
+
```
|
|
378
|
+
|
|
379
|
+
The [test](./test) directory contains the tests and the [input](./test/data)
|
|
380
|
+
files.
|
|
381
|
+
|
|
382
|
+
The code coverage is printed at the end using `simplecov`.
|
|
383
|
+
|
|
384
|
+
## Known Issues
|
|
385
|
+
|
|
386
|
+
- This plugin internally uses the HTTP server plugin
|
|
387
|
+
[helper](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-http_server)
|
|
388
|
+
which has higher precedence for `async-http` over `webrick`. But,
|
|
389
|
+
[`webrick`](https://github.com/ruby/webrick) is required to run this. In an
|
|
390
|
+
environment where both are installed, `async-http` is automatically selected
|
|
391
|
+
causing runtime issues. To make this work, you need to uninstall `async-http`
|
|
392
|
+
i.e. `gem uninstall async-http`. See issue
|
|
393
|
+
[#10](https://github.com/iamazeem/fluent-plugin-protobuf-http/issues/10) for
|
|
394
|
+
more details where this was identified in Docker containers where both gems
|
|
395
|
+
were already installed.
|
|
367
396
|
|
|
368
397
|
## Contribute
|
|
369
398
|
|
|
370
|
-
- Fork the project.
|
|
399
|
+
- [Fork](https://github.com/iamazeem/fluent-plugin-protobuf-http/fork) the project.
|
|
371
400
|
- Check out the latest `main` branch.
|
|
372
|
-
- Create a feature or bugfix branch from `main`.
|
|
401
|
+
- Create a `feature` or `bugfix` branch from `main`.
|
|
373
402
|
- Commit and push your changes.
|
|
374
403
|
- Make sure to add and run tests locally: `bundle exec rake test`.
|
|
375
|
-
- Run
|
|
404
|
+
- Run [Rubocop](https://github.com/rubocop/rubocop) and fix the lint errors.
|
|
376
405
|
- Submit the PR.
|
|
377
406
|
|
|
378
407
|
## License
|
|
@@ -5,7 +5,8 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
5
5
|
|
|
6
6
|
Gem::Specification.new do |spec|
|
|
7
7
|
spec.name = 'fluent-plugin-protobuf-http'
|
|
8
|
-
spec.version = '0.3.
|
|
8
|
+
spec.version = '0.3.1'
|
|
9
|
+
spec.date = '2023-09-16'
|
|
9
10
|
spec.authors = ['Azeem Sajid']
|
|
10
11
|
spec.email = ['azeem.sajid@gmail.com']
|
|
11
12
|
|
|
@@ -116,9 +116,10 @@ module Fluent
|
|
|
116
116
|
log.debug("Extracting message types [#{compiled_proto}]...")
|
|
117
117
|
msg_types = []
|
|
118
118
|
File.foreach(compiled_proto) do |line|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
119
|
+
line.strip!
|
|
120
|
+
if line.include?('::Google::Protobuf::DescriptorPool.generated_pool.lookup') && line.end_with?('.msgclass')
|
|
121
|
+
extracted_msg_type = line[/"([^"]*)"/, 1].freeze
|
|
122
|
+
msg_types.push(extracted_msg_type) unless extracted_msg_type.nil?
|
|
122
123
|
end
|
|
123
124
|
end
|
|
124
125
|
|
metadata
CHANGED
|
@@ -1,35 +1,35 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-protobuf-http
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Azeem Sajid
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2023-09-16 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
|
16
16
|
requirements:
|
|
17
|
-
- - "~>"
|
|
18
|
-
- !ruby/object:Gem::Version
|
|
19
|
-
version: '2.1'
|
|
20
17
|
- - ">="
|
|
21
18
|
- !ruby/object:Gem::Version
|
|
22
19
|
version: 2.1.0
|
|
20
|
+
- - "~>"
|
|
21
|
+
- !ruby/object:Gem::Version
|
|
22
|
+
version: '2.1'
|
|
23
23
|
type: :development
|
|
24
24
|
prerelease: false
|
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
|
26
26
|
requirements:
|
|
27
|
-
- - "~>"
|
|
28
|
-
- !ruby/object:Gem::Version
|
|
29
|
-
version: '2.1'
|
|
30
27
|
- - ">="
|
|
31
28
|
- !ruby/object:Gem::Version
|
|
32
29
|
version: 2.1.0
|
|
30
|
+
- - "~>"
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: '2.1'
|
|
33
33
|
- !ruby/object:Gem::Dependency
|
|
34
34
|
name: rake
|
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -126,6 +126,7 @@ executables: []
|
|
|
126
126
|
extensions: []
|
|
127
127
|
extra_rdoc_files: []
|
|
128
128
|
files:
|
|
129
|
+
- ".github/dependabot.yml"
|
|
129
130
|
- ".github/workflows/ci.yml"
|
|
130
131
|
- ".gitignore"
|
|
131
132
|
- ".rubocop.yml"
|
|
@@ -160,8 +161,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
160
161
|
- !ruby/object:Gem::Version
|
|
161
162
|
version: '0'
|
|
162
163
|
requirements: []
|
|
163
|
-
|
|
164
|
-
rubygems_version: 2.7.6
|
|
164
|
+
rubygems_version: 3.0.9
|
|
165
165
|
signing_key:
|
|
166
166
|
specification_version: 4
|
|
167
167
|
summary: fluentd HTTP Input Plugin for Protocol Buffers
|