fluent-plugin-formatter-protobuf 0.0.2 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/build-and-test.yml +54 -0
- data/.github/workflows/codeql-analysis.yml +49 -0
- data/.github/workflows/publish.yml +96 -0
- data/.gitignore +2 -2
- data/.rubocop.yml +7 -4
- data/.ruby-version +1 -1
- data/README.md +62 -8
- data/RELEASE.md +9 -0
- data/Rakefile +5 -0
- data/fluent-plugin-formatter-protobuf.gemspec +7 -5
- data/lib/fluent/plugin/formatter_protobuf.rb +14 -20
- data/lib/fluent/plugin/version.rb +1 -3
- data/test/integration/Gemfile +6 -0
- data/{Gemfile.lock → test/integration/Gemfile.lock} +7 -15
- data/test/integration/README.md +12 -0
- data/test/integration/fluent.conf +25 -0
- data/test/integration/log.proto +6 -0
- data/test/integration/log_pb.rb +15 -0
- data/test/integration/run.sh +12 -0
- data/test/plugin/test_formatter_protobuf.rb +22 -6
- metadata +57 -14
- data/.github/workflows/gem-push.yml +0 -19
- data/package.json +0 -58
- data/yarn.lock +0 -3725
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f49ca0a97713b1a0771cb9c73b38f49660b5fb4103a6d3f989140013fbd21bab
|
|
4
|
+
data.tar.gz: d547aaeef78fb13b4d5747dcf26fa27f1912e33ee6b4f699db401f335525fa94
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c17daaf911ea6a58cdbd8bd39b25bdf500630a506ab29f384d1abcf01e178578bea74ecf4e9ebff6c0496bf67304324f49b00744d49911bd1db32177d7fbcaed
|
|
7
|
+
data.tar.gz: bc2167f0f05d20594b453a7a40ba25c8e72ae085ed6c72a14714e2cdf617e9050f650b1c1aa1b82b5626f2c83666ef4015c19149deed227c4e0d39d79df910f2
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
name: Build and test
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
lint:
|
|
9
|
+
name: Lint
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
permissions:
|
|
12
|
+
packages: write
|
|
13
|
+
contents: read
|
|
14
|
+
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v2
|
|
17
|
+
- name: Set up Ruby 2.5
|
|
18
|
+
uses: ruby/setup-ruby@v1
|
|
19
|
+
with:
|
|
20
|
+
bundler-cache: true
|
|
21
|
+
- name: "Unit test"
|
|
22
|
+
run: "bundle exec rake lint:check"
|
|
23
|
+
|
|
24
|
+
test:
|
|
25
|
+
name: Unit test
|
|
26
|
+
runs-on: ubuntu-latest
|
|
27
|
+
permissions:
|
|
28
|
+
packages: write
|
|
29
|
+
contents: read
|
|
30
|
+
|
|
31
|
+
steps:
|
|
32
|
+
- uses: actions/checkout@v2
|
|
33
|
+
- name: Set up Ruby 2.5
|
|
34
|
+
uses: ruby/setup-ruby@v1
|
|
35
|
+
with:
|
|
36
|
+
bundler-cache: true
|
|
37
|
+
- name: "Unit test"
|
|
38
|
+
run: "bundle exec rake test:unit"
|
|
39
|
+
|
|
40
|
+
build:
|
|
41
|
+
name: Build
|
|
42
|
+
runs-on: ubuntu-latest
|
|
43
|
+
permissions:
|
|
44
|
+
packages: write
|
|
45
|
+
contents: read
|
|
46
|
+
|
|
47
|
+
steps:
|
|
48
|
+
- uses: actions/checkout@v2
|
|
49
|
+
- name: Set up Ruby 2.5
|
|
50
|
+
uses: ruby/setup-ruby@v1
|
|
51
|
+
with:
|
|
52
|
+
bundler-cache: true
|
|
53
|
+
- name: "Unit test"
|
|
54
|
+
run: "bundle exec rake build"
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
name: "CodeQL"
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [ main ]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [ main ]
|
|
8
|
+
schedule:
|
|
9
|
+
- cron: '18 11 * * 4'
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
analyze:
|
|
13
|
+
name: Analyze
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
permissions:
|
|
16
|
+
actions: read
|
|
17
|
+
contents: read
|
|
18
|
+
security-events: write
|
|
19
|
+
|
|
20
|
+
strategy:
|
|
21
|
+
fail-fast: false
|
|
22
|
+
matrix:
|
|
23
|
+
language: [ 'ruby' ]
|
|
24
|
+
|
|
25
|
+
steps:
|
|
26
|
+
- name: Checkout repository
|
|
27
|
+
uses: actions/checkout@v2
|
|
28
|
+
|
|
29
|
+
- name: Initialize CodeQL
|
|
30
|
+
uses: github/codeql-action/init@v1
|
|
31
|
+
with:
|
|
32
|
+
languages: ${{ matrix.language }}
|
|
33
|
+
|
|
34
|
+
- name: Autobuild
|
|
35
|
+
uses: github/codeql-action/autobuild@v1
|
|
36
|
+
|
|
37
|
+
# ℹ️ Command-line programs to run using the OS shell.
|
|
38
|
+
# 📚 https://git.io/JvXDl
|
|
39
|
+
|
|
40
|
+
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
|
|
41
|
+
# and modify them (or add more) to build your code if your project
|
|
42
|
+
# uses a compiled language
|
|
43
|
+
|
|
44
|
+
#- run: |
|
|
45
|
+
# make bootstrap
|
|
46
|
+
# make release
|
|
47
|
+
|
|
48
|
+
- name: Perform CodeQL Analysis
|
|
49
|
+
uses: github/codeql-action/analyze@v1
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
name: Publish
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- v0.*
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
lint:
|
|
10
|
+
name: Lint
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
permissions:
|
|
13
|
+
packages: write
|
|
14
|
+
contents: read
|
|
15
|
+
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v2
|
|
18
|
+
- name: Set up Ruby 2.5
|
|
19
|
+
uses: ruby/setup-ruby@v1
|
|
20
|
+
with:
|
|
21
|
+
bundler-cache: true
|
|
22
|
+
- name: "Unit test"
|
|
23
|
+
run: "bundle exec rake lint:check"
|
|
24
|
+
|
|
25
|
+
test:
|
|
26
|
+
name: Unit test
|
|
27
|
+
runs-on: ubuntu-latest
|
|
28
|
+
permissions:
|
|
29
|
+
packages: write
|
|
30
|
+
contents: read
|
|
31
|
+
|
|
32
|
+
steps:
|
|
33
|
+
- uses: actions/checkout@v2
|
|
34
|
+
- name: Set up Ruby 2.5
|
|
35
|
+
uses: ruby/setup-ruby@v1
|
|
36
|
+
with:
|
|
37
|
+
bundler-cache: true
|
|
38
|
+
- name: "Unit test"
|
|
39
|
+
run: "bundle exec rake test:unit"
|
|
40
|
+
|
|
41
|
+
publish-to-gpr:
|
|
42
|
+
needs:
|
|
43
|
+
- lint
|
|
44
|
+
- test
|
|
45
|
+
name: Build + Publish to GPR
|
|
46
|
+
runs-on: ubuntu-latest
|
|
47
|
+
permissions:
|
|
48
|
+
packages: write
|
|
49
|
+
contents: read
|
|
50
|
+
|
|
51
|
+
steps:
|
|
52
|
+
- uses: actions/checkout@v2
|
|
53
|
+
- name: Set up Ruby 2.5
|
|
54
|
+
uses: ruby/setup-ruby@v1
|
|
55
|
+
with:
|
|
56
|
+
bundler-cache: true
|
|
57
|
+
|
|
58
|
+
- name: Publish to GPR
|
|
59
|
+
run: |
|
|
60
|
+
mkdir -p $HOME/.gem
|
|
61
|
+
touch $HOME/.gem/credentials
|
|
62
|
+
chmod 0600 $HOME/.gem/credentials
|
|
63
|
+
printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
|
|
64
|
+
gem build *.gemspec
|
|
65
|
+
gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} *.gem
|
|
66
|
+
env:
|
|
67
|
+
GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
|
|
68
|
+
OWNER: ${{ github.repository_owner }}
|
|
69
|
+
|
|
70
|
+
publish-to-ruby-gems:
|
|
71
|
+
needs:
|
|
72
|
+
- lint
|
|
73
|
+
- test
|
|
74
|
+
name: Build + Publish to RubyGems
|
|
75
|
+
runs-on: ubuntu-latest
|
|
76
|
+
permissions:
|
|
77
|
+
packages: write
|
|
78
|
+
contents: read
|
|
79
|
+
|
|
80
|
+
steps:
|
|
81
|
+
- uses: actions/checkout@v2
|
|
82
|
+
- name: Set up Ruby 2.5
|
|
83
|
+
uses: ruby/setup-ruby@v1
|
|
84
|
+
with:
|
|
85
|
+
bundler-cache: true
|
|
86
|
+
|
|
87
|
+
- name: Publish to RubyGems
|
|
88
|
+
run: |
|
|
89
|
+
mkdir -p $HOME/.gem
|
|
90
|
+
touch $HOME/.gem/credentials
|
|
91
|
+
chmod 0600 $HOME/.gem/credentials
|
|
92
|
+
printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
|
|
93
|
+
gem build *.gemspec
|
|
94
|
+
gem push *.gem
|
|
95
|
+
env:
|
|
96
|
+
GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"
|
data/.gitignore
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
*.gem
|
|
2
|
+
Gemfile.lock
|
data/.rubocop.yml
CHANGED
|
@@ -1,13 +1,16 @@
|
|
|
1
|
+
require: rubocop-rake
|
|
1
2
|
AllCops:
|
|
2
3
|
NewCops: enable
|
|
3
4
|
Style/StringLiterals:
|
|
4
5
|
Exclude:
|
|
5
|
-
- '
|
|
6
|
+
- '**/*_pb.rb'
|
|
6
7
|
Style/HashSyntax:
|
|
7
8
|
Exclude:
|
|
8
|
-
- '
|
|
9
|
+
- '**/*_pb.rb'
|
|
9
10
|
Style/FrozenStringLiteralComment:
|
|
10
11
|
Exclude:
|
|
11
|
-
- '
|
|
12
|
+
- '**/*_pb.rb'
|
|
12
13
|
Metrics/MethodLength:
|
|
13
|
-
Max: 100
|
|
14
|
+
Max: 100
|
|
15
|
+
Exclude:
|
|
16
|
+
- '**/*_pb.rb'
|
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
2.
|
|
1
|
+
2.5.0
|
data/README.md
CHANGED
|
@@ -1,21 +1,40 @@
|
|
|
1
1
|
# fluent-plugin-formatter-protobuf
|
|
2
2
|
|
|
3
|
+
[](https://badge.fury.io/rb/fluent-plugin-formatter-protobuf)
|
|
4
|
+
|
|
3
5
|
[Fluentd](https://fluentd.org/) formatter plugin to format messages into Protobuf v3 encoded binary.
|
|
4
6
|
|
|
5
7
|
## Installation
|
|
6
8
|
|
|
7
9
|
### RubyGems
|
|
8
10
|
|
|
9
|
-
```
|
|
11
|
+
```shell
|
|
10
12
|
$ gem install fluent-plugin-formatter-protobuf
|
|
11
13
|
```
|
|
12
14
|
|
|
13
|
-
###
|
|
15
|
+
### GitHub RubyGems Registry
|
|
16
|
+
|
|
17
|
+
```shell
|
|
18
|
+
$ gem install \
|
|
19
|
+
fluent-plugin-formatter-protobuf \
|
|
20
|
+
--version "<version>" \
|
|
21
|
+
--source "https://rubygems.pkg.github.com/fluent-plugins-nursery"
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
### Bundler (RubyGems)
|
|
14
25
|
|
|
15
26
|
Add following line to your Gemfile:
|
|
16
27
|
|
|
17
28
|
```ruby
|
|
18
|
-
gem "fluent-plugin-formatter-protobuf"
|
|
29
|
+
gem "fluent-plugin-formatter-protobuf", "<version>"
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
### Bundler (GitHub RubyGems Registry),
|
|
33
|
+
|
|
34
|
+
```shell
|
|
35
|
+
source "https://rubygems.pkg.github.com/fluent-plugins-nursery" do
|
|
36
|
+
gem "fluent-plugin-formatter-protobuf", "<version>"
|
|
37
|
+
end
|
|
19
38
|
```
|
|
20
39
|
|
|
21
40
|
And then execute:
|
|
@@ -24,15 +43,50 @@ And then execute:
|
|
|
24
43
|
$ bundle
|
|
25
44
|
```
|
|
26
45
|
|
|
27
|
-
##
|
|
46
|
+
## Quick start
|
|
28
47
|
|
|
29
|
-
|
|
48
|
+
1. Generate the protobuf ruby methods
|
|
49
|
+
1. e.g `protoc --proto_path=. --ruby_out=. ./log.proto`
|
|
50
|
+
2. Modify the `<format>` section to include `class_name`, which is your Protobuf message name, and `include_paths`, the path where the generated Ruby types are stored
|
|
51
|
+
1. Given protobuf class `Your::Protobuf::Class::Name` class should be given as `Your.Protobuf.Class.Name` in `class_name`. The exact name can be found in the generated Ruby files
|
|
30
52
|
|
|
53
|
+
|
|
54
|
+
## Example
|
|
55
|
+
|
|
56
|
+
```fluentd
|
|
57
|
+
<source>
|
|
58
|
+
@type tcp
|
|
59
|
+
tag tcp.events
|
|
60
|
+
port 5170
|
|
61
|
+
delimiter "\n"
|
|
62
|
+
<parse>
|
|
63
|
+
@type json
|
|
64
|
+
</parse>
|
|
65
|
+
</source>
|
|
66
|
+
|
|
67
|
+
<match tcp.events>
|
|
68
|
+
@type file
|
|
69
|
+
|
|
70
|
+
path /opt/fluent-plugin-formatter-protobuf/out
|
|
71
|
+
|
|
72
|
+
<buffer>
|
|
73
|
+
@type memory
|
|
74
|
+
</buffer>
|
|
75
|
+
|
|
76
|
+
<format>
|
|
77
|
+
@type protobuf
|
|
78
|
+
class_name "Log"
|
|
79
|
+
include_paths ["/opt/fluent-plugin-formatter-protobuf/log_pb.rb"]
|
|
80
|
+
</format>
|
|
81
|
+
</match>
|
|
31
82
|
```
|
|
32
|
-
$ fluent-plugin-config-format formatter formatter-protobuf
|
|
33
|
-
```
|
|
34
83
|
|
|
35
|
-
|
|
84
|
+
## Configuration
|
|
85
|
+
|
|
86
|
+
|parameter|type|description|default|
|
|
87
|
+
|---|---|---|---|
|
|
88
|
+
|include_paths|array (optional)| Generated Ruby Protobuf class files path |`[]`|
|
|
89
|
+
|class_name|string (required)| Ruby Protobuf class name. Used to encode into Protobuf binary||
|
|
36
90
|
|
|
37
91
|
## Copyright
|
|
38
92
|
|
data/RELEASE.md
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Release
|
|
2
|
+
|
|
3
|
+
## Instructions
|
|
4
|
+
To release the gem to Github package manager and Ruby Gems,
|
|
5
|
+
|
|
6
|
+
1. Update `lib/fluent/plugin/version.rb`
|
|
7
|
+
2. Create a pull request and merge it into `main`
|
|
8
|
+
3. Manually creating a release on Github with a tag indicating the new version number e.g. `vMAJOR.MINOR.PATCH`
|
|
9
|
+
4. GitHub Actions will automatically run and publish to RubyGems and GitHub repository
|
data/Rakefile
CHANGED
|
@@ -6,7 +6,7 @@ require 'fluent/plugin/version'
|
|
|
6
6
|
|
|
7
7
|
Gem::Specification.new do |spec|
|
|
8
8
|
spec.name = 'fluent-plugin-formatter-protobuf'
|
|
9
|
-
spec.version = Fluent::Plugin::
|
|
9
|
+
spec.version = Fluent::Plugin::VERSION
|
|
10
10
|
spec.authors = ['Ray Tung']
|
|
11
11
|
spec.email = ['code@raytung.net']
|
|
12
12
|
|
|
@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
|
|
|
14
14
|
spec.description = 'This is a Fluentd formatter plugin designed to convert Protobuf JSON into Protobuf binary'
|
|
15
15
|
spec.homepage = 'https://github.com/raytung/fluent-plugin-formatter-protobuf'
|
|
16
16
|
spec.license = 'Apache-2.0'
|
|
17
|
-
spec.required_ruby_version = Gem::Requirement.new('>=2.
|
|
17
|
+
spec.required_ruby_version = Gem::Requirement.new('>= 2.5.0')
|
|
18
18
|
|
|
19
19
|
spec.metadata['homepage_uri'] = spec.homepage
|
|
20
20
|
spec.metadata['source_code_uri'] = 'https://github.com/raytung/fluent-plugin-formatter-protobuf'
|
|
@@ -28,9 +28,11 @@ Gem::Specification.new do |spec|
|
|
|
28
28
|
spec.test_files = test_files
|
|
29
29
|
spec.require_paths = ['lib']
|
|
30
30
|
|
|
31
|
-
spec.add_development_dependency 'bundler', '~> 2.2
|
|
32
|
-
spec.add_development_dependency 'rake', '~> 13.0
|
|
33
|
-
spec.add_development_dependency '
|
|
31
|
+
spec.add_development_dependency 'bundler', '~> 2.2'
|
|
32
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
|
33
|
+
spec.add_development_dependency 'rubocop', '~> 1.22'
|
|
34
|
+
spec.add_development_dependency 'rubocop-rake', '~> 0.6'
|
|
35
|
+
spec.add_development_dependency 'test-unit', '~> 3.3'
|
|
34
36
|
spec.add_runtime_dependency 'fluentd', ['>= 1.0', '< 2']
|
|
35
37
|
spec.add_runtime_dependency 'google-protobuf', ['~> 3.18']
|
|
36
38
|
end
|
|
@@ -29,11 +29,9 @@ module Fluent
|
|
|
29
29
|
class ProtobufFormatter < Fluent::Plugin::Formatter
|
|
30
30
|
Fluent::Plugin.register_formatter('protobuf', self)
|
|
31
31
|
|
|
32
|
-
|
|
33
|
-
config_param :include_paths, :array, default: []
|
|
32
|
+
config_param :include_paths, :array, default: [], desc: 'Generated Ruby Protobuf class files path'
|
|
34
33
|
|
|
35
|
-
|
|
36
|
-
config_param :message_name, :string
|
|
34
|
+
config_param :class_name, :string, desc: 'Ruby Protobuf class name. Used to encode into Protobuf binary'
|
|
37
35
|
|
|
38
36
|
def configure(conf)
|
|
39
37
|
super(conf)
|
|
@@ -42,10 +40,14 @@ module Fluent
|
|
|
42
40
|
|
|
43
41
|
@include_paths.each { |path| require_proto!(path) } unless @include_paths.empty?
|
|
44
42
|
|
|
45
|
-
|
|
46
|
-
raise Fluent::ConfigError, "
|
|
43
|
+
class_lookup = Google::Protobuf::DescriptorPool.generated_pool.lookup(@class_name)
|
|
44
|
+
raise Fluent::ConfigError, "class name '#{@class_name}' not found" if class_lookup.nil?
|
|
47
45
|
|
|
48
|
-
@protobuf_class =
|
|
46
|
+
@protobuf_class = class_lookup.msgclass
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def formatter_type
|
|
50
|
+
:binary
|
|
49
51
|
end
|
|
50
52
|
|
|
51
53
|
def format(_tag, _time, record)
|
|
@@ -53,22 +55,14 @@ module Fluent
|
|
|
53
55
|
@protobuf_class.encode(protobuf_msg)
|
|
54
56
|
end
|
|
55
57
|
|
|
56
|
-
# @param [string] filename
|
|
57
|
-
# @return [void]
|
|
58
58
|
def require_proto!(filename)
|
|
59
|
-
|
|
60
|
-
raise Fluent::ConfigError, "Unable to load file '#{filename}'. It is not a Ruby file"
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
unless Pathname.new(filename).absolute?
|
|
64
|
-
raise Fluent::ConfigError, "Unable to load file '#{filename}'. Please provide absolute paths"
|
|
65
|
-
end
|
|
66
|
-
|
|
67
|
-
begin
|
|
59
|
+
if Pathname.new(filename).absolute?
|
|
68
60
|
require filename
|
|
69
|
-
|
|
70
|
-
|
|
61
|
+
else
|
|
62
|
+
require_relative filename
|
|
71
63
|
end
|
|
64
|
+
rescue LoadError => e
|
|
65
|
+
raise Fluent::ConfigError, "Unable to load file '#{filename}'. Reason: #{e.inspect}"
|
|
72
66
|
end
|
|
73
67
|
end
|
|
74
68
|
end
|
|
@@ -1,15 +1,11 @@
|
|
|
1
|
-
PATH
|
|
2
|
-
remote: .
|
|
3
|
-
specs:
|
|
4
|
-
fluent-plugin-formatter-protobuf (0.0.2)
|
|
5
|
-
fluentd (>= 1.0, < 2)
|
|
6
|
-
google-protobuf (~> 3.18)
|
|
7
|
-
|
|
8
1
|
GEM
|
|
9
2
|
remote: https://rubygems.org/
|
|
10
3
|
specs:
|
|
11
4
|
concurrent-ruby (1.1.9)
|
|
12
5
|
cool.io (1.7.1)
|
|
6
|
+
fluent-plugin-formatter-protobuf (0.0.3)
|
|
7
|
+
fluentd (>= 1.0, < 2)
|
|
8
|
+
google-protobuf (~> 3.18)
|
|
13
9
|
fluentd (1.14.1)
|
|
14
10
|
bundler
|
|
15
11
|
cool.io (>= 1.4.5, < 2.0.0)
|
|
@@ -22,17 +18,14 @@ GEM
|
|
|
22
18
|
tzinfo-data (~> 1.0)
|
|
23
19
|
webrick (>= 1.4.2, < 1.8.0)
|
|
24
20
|
yajl-ruby (~> 1.0)
|
|
21
|
+
google-protobuf (3.18.1)
|
|
25
22
|
google-protobuf (3.18.1-x86_64-linux)
|
|
26
23
|
http_parser.rb (0.7.0)
|
|
27
24
|
msgpack (1.4.2)
|
|
28
|
-
power_assert (2.0.1)
|
|
29
|
-
rake (13.0.6)
|
|
30
25
|
serverengine (2.2.4)
|
|
31
26
|
sigdump (~> 0.2.2)
|
|
32
27
|
sigdump (0.2.4)
|
|
33
28
|
strptime (0.2.5)
|
|
34
|
-
test-unit (3.3.9)
|
|
35
|
-
power_assert
|
|
36
29
|
tzinfo (2.0.4)
|
|
37
30
|
concurrent-ruby (~> 1.0)
|
|
38
31
|
tzinfo-data (1.2021.3)
|
|
@@ -41,13 +34,12 @@ GEM
|
|
|
41
34
|
yajl-ruby (1.4.1)
|
|
42
35
|
|
|
43
36
|
PLATFORMS
|
|
37
|
+
ruby
|
|
44
38
|
x86_64-linux
|
|
45
39
|
|
|
46
40
|
DEPENDENCIES
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
rake (~> 13.0.3)
|
|
50
|
-
test-unit (~> 3.3.7)
|
|
41
|
+
fluent-plugin-formatter-protobuf (= 0.0.3)
|
|
42
|
+
fluentd (= 1.14.1)
|
|
51
43
|
|
|
52
44
|
BUNDLED WITH
|
|
53
45
|
2.2.29
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Integration test
|
|
2
|
+
|
|
3
|
+
A _manual_ integration test.
|
|
4
|
+
|
|
5
|
+
### Instructions
|
|
6
|
+
|
|
7
|
+
1. Open a new terminal and run`$ ./run.sh`
|
|
8
|
+
2. Open a separate terminal and run `echo '{"timestamp": 1634386250, "message":"hello!"}\n' | netcat 127.0.0.1 5170`
|
|
9
|
+
3. Somehow flush data (I simply stop the docker container)
|
|
10
|
+
4. Verify out data (It should look like `out.YYYYMMDD_N.log`) with `protoc`
|
|
11
|
+
1. `protoc --decode_raw < out.YYYYMMDD_N.log`
|
|
12
|
+
2. `cat out.YYYYMMDD_N.log | protoc --decode=Log --proto_path=. log.proto`
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
<source>
|
|
2
|
+
@type tcp
|
|
3
|
+
tag tcp.events
|
|
4
|
+
port 5170
|
|
5
|
+
delimiter "\n"
|
|
6
|
+
<parse>
|
|
7
|
+
@type json
|
|
8
|
+
</parse>
|
|
9
|
+
</source>
|
|
10
|
+
|
|
11
|
+
<match tcp.events>
|
|
12
|
+
@type file
|
|
13
|
+
|
|
14
|
+
path /opt/fluent-plugin-formatter-protobuf/out
|
|
15
|
+
|
|
16
|
+
<buffer>
|
|
17
|
+
@type memory
|
|
18
|
+
</buffer>
|
|
19
|
+
|
|
20
|
+
<format>
|
|
21
|
+
@type protobuf
|
|
22
|
+
message_name "Log"
|
|
23
|
+
include_paths ["/opt/fluent-plugin-formatter-protobuf/log_pb.rb"]
|
|
24
|
+
</format>
|
|
25
|
+
</match>
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
2
|
+
# source: log.proto
|
|
3
|
+
|
|
4
|
+
require 'google/protobuf'
|
|
5
|
+
|
|
6
|
+
Google::Protobuf::DescriptorPool.generated_pool.build do
|
|
7
|
+
add_file("log.proto", :syntax => :proto3) do
|
|
8
|
+
add_message "Log" do
|
|
9
|
+
optional :timestamp, :int64, 1
|
|
10
|
+
optional :message, :string, 2
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
Log = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("Log").msgclass
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
3
|
+
set -euo pipefail
|
|
4
|
+
|
|
5
|
+
cd "$(dirname "$0")"
|
|
6
|
+
|
|
7
|
+
docker run -it --rm \
|
|
8
|
+
-v "${PWD}:/opt/fluent-plugin-formatter-protobuf/" \
|
|
9
|
+
-w '/opt/fluent-plugin-formatter-protobuf/' \
|
|
10
|
+
-e OUT_FILE="./out" \
|
|
11
|
+
-p 5170:5170 \
|
|
12
|
+
ruby:2.7.1 sh -c 'bundle install && fluentd -c /opt/fluent-plugin-formatter-protobuf/fluent.conf'
|
|
@@ -8,31 +8,47 @@ class ProtobufFormatterTest < Test::Unit::TestCase
|
|
|
8
8
|
Fluent::Test.setup
|
|
9
9
|
end
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
VALID_INCLUDE_PATHS_ABSOLUTE = [File.expand_path(File.join(__dir__, '..', 'proto', 'addressbook_pb.rb'))].freeze
|
|
12
|
+
|
|
13
|
+
# Relative to the plugin file
|
|
14
|
+
VALID_INCLUDE_PATHS_RELATIVE = '../../../test/proto/addressbook_pb.rb'
|
|
12
15
|
|
|
13
16
|
sub_test_case 'configure' do
|
|
14
17
|
test 'fail if include_paths is empty' do
|
|
15
18
|
assert_raise(Fluent::ConfigError) do
|
|
16
|
-
create_driver({
|
|
19
|
+
create_driver({ class_name: '', include_paths: [] })
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
test 'fail if ruby files not found in the provided include paths' do
|
|
24
|
+
assert_raise(Fluent::ConfigError) do
|
|
25
|
+
create_driver({ class_name: 'tutorial.AddressBook', include_paths: ['some/random/path'] })
|
|
17
26
|
end
|
|
18
27
|
end
|
|
19
28
|
|
|
20
|
-
test 'fail if no protobuf class can be found with
|
|
29
|
+
test 'fail if no protobuf class can be found with class_name' do
|
|
21
30
|
assert_raise(Fluent::ConfigError) do
|
|
22
|
-
create_driver({
|
|
31
|
+
create_driver({ class_name: 'Some.Name', include_paths: VALID_INCLUDE_PATHS_ABSOLUTE })
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
test 'success if given valid relative paths in include paths' do
|
|
36
|
+
assert_nothing_raised do
|
|
37
|
+
create_driver({ class_name: 'tutorial.AddressBook', include_paths: [VALID_INCLUDE_PATHS_RELATIVE] })
|
|
23
38
|
end
|
|
24
39
|
end
|
|
25
40
|
|
|
26
41
|
test 'passes on valid configuration' do
|
|
27
42
|
assert_nothing_raised do
|
|
28
|
-
create_driver({
|
|
43
|
+
create_driver({ class_name: 'tutorial.AddressBook', include_paths: VALID_INCLUDE_PATHS_ABSOLUTE })
|
|
29
44
|
end
|
|
30
45
|
end
|
|
31
46
|
end
|
|
32
47
|
|
|
33
48
|
sub_test_case 'format' do
|
|
34
49
|
test 'encodes into Protobuf binary' do
|
|
35
|
-
formatter = create_formatter({
|
|
50
|
+
formatter = create_formatter({ class_name: 'tutorial.AddressBook',
|
|
51
|
+
include_paths: VALID_INCLUDE_PATHS_ABSOLUTE })
|
|
36
52
|
|
|
37
53
|
formatted = formatter.format('some-tag', 1234,
|
|
38
54
|
{ people: [{ name: 'Masahiro', id: 1337, email: 'repeatedly _at_ gmail.com' }] })
|