fluent-plugin-filter-split-message 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.devcontainer/Dockerfile +10 -0
- data/.devcontainer/devcontainer.json +17 -0
- data/.devcontainer/post-create.sh +6 -0
- data/.github/workflows/build-and-test.yml +26 -0
- data/.github/workflows/publish.yml +23 -0
- data/.gitignore +1 -0
- data/.vscode/settings.json +21 -0
- data/Gemfile +3 -0
- data/LICENSE +22 -0
- data/README.md +45 -0
- data/Rakefile +13 -0
- data/fluent-plugin-filter-split-message.gemspec +28 -0
- data/lib/fluent/plugin/filter_split_message.rb +47 -0
- data/test/test_split.rb +87 -0
- metadata +114 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: fd65ccd43f2d723119a6b1ac0285296396adf742b384fce09b682d36f3d2872e
|
4
|
+
data.tar.gz: e4954f1d238d4dfffc5251f79d04a515a64975d96e76a796644f910967d36db6
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 15d4a54ce99d94390d977049eec2bdd84c2e6f79d381b04ad9ff920258e1b0714973df3f096f08004af60880d40c26efbff1adc8c06c1ff6d91c9838344e980d
|
7
|
+
data.tar.gz: d68bb390e637ac241c9bd77d0d29228047b8f88d3fe133b5aa34ce702a44d90bc7b343f08e8d7e1de3c2ee116965c5351233f490a8bd5980f6c962b4845d516d
|
@@ -0,0 +1,17 @@
|
|
1
|
+
{
|
2
|
+
"name": "Fluent Plugin Filter Message Splitter",
|
3
|
+
"build": {
|
4
|
+
"dockerfile": "Dockerfile"
|
5
|
+
},
|
6
|
+
"customizations": {
|
7
|
+
"vscode":{
|
8
|
+
"extensions": [
|
9
|
+
"rebornix.Ruby",
|
10
|
+
"castwide.solargraph",
|
11
|
+
"jnbt.vscode-rufo"
|
12
|
+
]
|
13
|
+
}
|
14
|
+
},
|
15
|
+
"remoteUser": "vscode",
|
16
|
+
"postCreateCommand": "bash -i .devcontainer/post-create.sh"
|
17
|
+
}
|
@@ -0,0 +1,26 @@
|
|
1
|
+
name: Build and Test
|
2
|
+
on:
|
3
|
+
- push
|
4
|
+
- pull_request
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
runs-on: ${{ matrix.os }}
|
8
|
+
strategy:
|
9
|
+
fail-fast: false
|
10
|
+
matrix:
|
11
|
+
ruby: [ '2.6', '2.7', '3.0' ]
|
12
|
+
os:
|
13
|
+
- ubuntu-latest
|
14
|
+
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
15
|
+
steps:
|
16
|
+
- uses: actions/checkout@v3
|
17
|
+
- uses: ruby/setup-ruby@v1
|
18
|
+
with:
|
19
|
+
ruby-version: ${{ matrix.ruby }}
|
20
|
+
- name: unit testing
|
21
|
+
env:
|
22
|
+
CI: true
|
23
|
+
run: |
|
24
|
+
gem install bundler rake
|
25
|
+
bundle install --jobs 4 --retry 3
|
26
|
+
bundle exec rake test
|
@@ -0,0 +1,23 @@
|
|
1
|
+
name: Publish Gem
|
2
|
+
|
3
|
+
on:
|
4
|
+
release:
|
5
|
+
types:
|
6
|
+
- created
|
7
|
+
|
8
|
+
jobs:
|
9
|
+
publish:
|
10
|
+
if: github.repository_owner == 'bitpatty'
|
11
|
+
runs-on: ubuntu-latest
|
12
|
+
environment: package-registries
|
13
|
+
permissions:
|
14
|
+
contents: read
|
15
|
+
packages: write
|
16
|
+
steps:
|
17
|
+
- name: Checkout
|
18
|
+
uses: actions/checkout@v3
|
19
|
+
- name: Publish gem
|
20
|
+
uses: dawidd6/action-publish-gem@v1
|
21
|
+
with:
|
22
|
+
api_key: ${{ secrets.RUBYGEMS_API_KEY }}
|
23
|
+
github_token: ${{ secrets.GITHUB_TOKEN }}
|
data/.gitignore
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
*.lock
|
@@ -0,0 +1,21 @@
|
|
1
|
+
{
|
2
|
+
"files.autoSave": "off",
|
3
|
+
"breadcrumbs.symbolSortOrder": "type",
|
4
|
+
"editor.codeLens": true,
|
5
|
+
"editor.detectIndentation": true,
|
6
|
+
"editor.formatOnSave": true,
|
7
|
+
"editor.minimap.maxColumn": 150,
|
8
|
+
"editor.tabSize": 2,
|
9
|
+
"explorer.confirmDragAndDrop": false,
|
10
|
+
"git.confirmSync": false,
|
11
|
+
"git.enableSmartCommit": true,
|
12
|
+
"html.format.wrapLineLength": 150,
|
13
|
+
"javascript.updateImportsOnFileMove.enabled": "always",
|
14
|
+
"typescript.preferences.importModuleSpecifier": "relative",
|
15
|
+
"typescript.referencesCodeLens.enabled": true,
|
16
|
+
"typescript.referencesCodeLens.showOnAllFunctions": true,
|
17
|
+
"typescript.reportStyleChecksAsWarnings": true,
|
18
|
+
"typescript.updateImportsOnFileMove.enabled": "always",
|
19
|
+
"workbench.editor.enablePreview": false,
|
20
|
+
"workbench.editor.enablePreviewFromQuickOpen": false
|
21
|
+
}
|
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
MIT License
|
2
|
+
|
3
|
+
Copyright (c) 2017 Active Shadow LLC
|
4
|
+
Copyright (c) 2023 Matteias Collet
|
5
|
+
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
7
|
+
of this software and associated documentation files (the "Software"), to deal
|
8
|
+
in the Software without restriction, including without limitation the rights
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
11
|
+
furnished to do so, subject to the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be included in all
|
14
|
+
copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
22
|
+
SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
# fluent-plugin-filter-split-message
|
2
|
+
|
3
|
+
A [fluentd](https://www.fluentd.org/) filter plugin for splitting messages by a customizable delimiter.
|
4
|
+
|
5
|
+
## Sample
|
6
|
+
|
7
|
+
```ruby
|
8
|
+
{"message": "abc,def,ghi"}
|
9
|
+
```
|
10
|
+
|
11
|
+
.. is turned into ..
|
12
|
+
|
13
|
+
```ruby
|
14
|
+
{"message": "abc"}
|
15
|
+
{"message": "def"}
|
16
|
+
{"message": "ghi"}
|
17
|
+
```
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
Install the plugin:
|
22
|
+
|
23
|
+
```sh
|
24
|
+
# See https://github.com/BitPatty/fluent-plugin-filter-split-message/releases for a list of valid versions
|
25
|
+
gem install fluent-plugin-filter-split-message --version "<desired version>"
|
26
|
+
```
|
27
|
+
|
28
|
+
Create a logdrain and update your fluent configuration:
|
29
|
+
|
30
|
+
```conf
|
31
|
+
<filter>
|
32
|
+
@type split_message
|
33
|
+
|
34
|
+
# (Optional) The delimiter to use, defaults to ","
|
35
|
+
delimiter ,
|
36
|
+
|
37
|
+
# (Optional) The target field, defaults to "message"
|
38
|
+
field_key message
|
39
|
+
</source>
|
40
|
+
```
|
41
|
+
|
42
|
+
|
43
|
+
## Credit
|
44
|
+
|
45
|
+
- [fluent-plugin-split-event](https://github.com/uken/fluent-plugin-split-event) used as plugin reference
|
data/Rakefile
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
require "bundler"
|
2
|
+
Bundler::GemHelper.install_tasks
|
3
|
+
|
4
|
+
require "rake/testtask"
|
5
|
+
|
6
|
+
Rake::TestTask.new(:test) do |t|
|
7
|
+
t.libs.push("lib", "test")
|
8
|
+
t.test_files = FileList["test/**/test_*.rb"]
|
9
|
+
t.verbose = true
|
10
|
+
t.warning = true
|
11
|
+
end
|
12
|
+
|
13
|
+
task default: [:test]
|
@@ -0,0 +1,28 @@
|
|
1
|
+
lib = File.expand_path("../lib", __FILE__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
|
4
|
+
Gem::Specification.new do |spec|
|
5
|
+
spec.name = "fluent-plugin-filter-split-message"
|
6
|
+
spec.version = "0.1.0"
|
7
|
+
spec.authors = ["Matteias Collet"]
|
8
|
+
spec.email = ["matteias.collet@protonmail.ch"]
|
9
|
+
|
10
|
+
spec.summary = %q{Message splitter filter for Fluentd}
|
11
|
+
spec.description = spec.summary
|
12
|
+
spec.homepage = "https://github.com/bitpatty/fluent-plugin-filter-split-message"
|
13
|
+
spec.license = "MIT"
|
14
|
+
|
15
|
+
test_files, files = `git ls-files -z`.split("\x0").partition do |f|
|
16
|
+
f.match(%r{^(test|spec|features)/})
|
17
|
+
end
|
18
|
+
|
19
|
+
spec.files = files
|
20
|
+
spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
|
21
|
+
spec.test_files = test_files
|
22
|
+
spec.require_paths = ["lib"]
|
23
|
+
|
24
|
+
spec.add_development_dependency "bundler", "~> 2.1"
|
25
|
+
spec.add_development_dependency "rake", "~> 13.0"
|
26
|
+
spec.add_development_dependency "test-unit", "~> 3.3"
|
27
|
+
spec.add_runtime_dependency "fluentd", ">= 1"
|
28
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require "fluent/plugin/filter"
|
2
|
+
require "fluent/time"
|
3
|
+
|
4
|
+
module Fluent
|
5
|
+
module Plugin
|
6
|
+
class SplitMessage < Filter
|
7
|
+
Plugin.register_filter("split_message", self)
|
8
|
+
|
9
|
+
config_param :field_key, :string, default: "message"
|
10
|
+
config_param :delimiter, :string, default: ","
|
11
|
+
|
12
|
+
def configure(conf)
|
13
|
+
super
|
14
|
+
end
|
15
|
+
|
16
|
+
def filter_stream(tag, es)
|
17
|
+
new_es = Fluent::MultiEventStream.new
|
18
|
+
|
19
|
+
es.each do |time, record|
|
20
|
+
begin
|
21
|
+
unless record.key?(@field_key)
|
22
|
+
return new_es
|
23
|
+
end
|
24
|
+
|
25
|
+
vals = record[@field_key].split(@delimiter)
|
26
|
+
|
27
|
+
if vals.count > 1
|
28
|
+
vals.each do |v|
|
29
|
+
begin
|
30
|
+
new_record = record.dup
|
31
|
+
new_record[@field_key] = v.strip
|
32
|
+
new_es.add(time, new_record)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
else
|
36
|
+
new_es.add(time, record)
|
37
|
+
end
|
38
|
+
rescue => e
|
39
|
+
router.emit_error_event(tag, time, record, e)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
return new_es
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
data/test/test_split.rb
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
require "test/unit"
|
2
|
+
require "fluent/test"
|
3
|
+
require "fluent/test/driver/filter"
|
4
|
+
require "fluent/plugin/filter_split_message.rb"
|
5
|
+
|
6
|
+
class SplitMessageTest < Test::Unit::TestCase
|
7
|
+
def setup
|
8
|
+
Fluent::Test.setup # this is required to setup router and others
|
9
|
+
end
|
10
|
+
|
11
|
+
def create_driver(conf = "")
|
12
|
+
Fluent::Test::Driver::Filter.new(Fluent::Plugin::SplitMessage).configure(conf)
|
13
|
+
end
|
14
|
+
|
15
|
+
def filter(config, messages)
|
16
|
+
d = create_driver(config)
|
17
|
+
d.run(default_tag: "test") do
|
18
|
+
messages.each do |message|
|
19
|
+
d.feed(message)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
d.filtered_records
|
23
|
+
end
|
24
|
+
|
25
|
+
test "empty message" do
|
26
|
+
messages = [
|
27
|
+
{ "message" => "" },
|
28
|
+
]
|
29
|
+
expected = [
|
30
|
+
{ "message" => "" },
|
31
|
+
]
|
32
|
+
filtered_records = filter("", messages)
|
33
|
+
assert_equal(expected, filtered_records)
|
34
|
+
end
|
35
|
+
|
36
|
+
test "simple split" do
|
37
|
+
messages = [
|
38
|
+
{ "message" => "foobar,abc" },
|
39
|
+
]
|
40
|
+
expected = [
|
41
|
+
{ "message" => "foobar" },
|
42
|
+
{ "message" => "abc" },
|
43
|
+
]
|
44
|
+
filtered_records = filter("", messages)
|
45
|
+
assert_equal(expected, filtered_records)
|
46
|
+
end
|
47
|
+
|
48
|
+
test "multi char split" do
|
49
|
+
messages = [
|
50
|
+
{ "message" => "foobar,,,,abc,,,," },
|
51
|
+
]
|
52
|
+
expected = [
|
53
|
+
{ "message" => "foobar" },
|
54
|
+
{ "message" => "abc" },
|
55
|
+
]
|
56
|
+
filtered_records = filter(%[
|
57
|
+
delimiter ',,,,'
|
58
|
+
], messages)
|
59
|
+
assert_equal(expected, filtered_records)
|
60
|
+
end
|
61
|
+
|
62
|
+
test "multi split" do
|
63
|
+
messages = [
|
64
|
+
{ "message" => "foobar,hugo,abc" },
|
65
|
+
]
|
66
|
+
expected = [
|
67
|
+
{ "message" => "foobar" },
|
68
|
+
{ "message" => "hugo" },
|
69
|
+
{ "message" => "abc" },
|
70
|
+
]
|
71
|
+
filtered_records = filter(%[], messages)
|
72
|
+
assert_equal(expected, filtered_records)
|
73
|
+
end
|
74
|
+
|
75
|
+
test "trim" do
|
76
|
+
messages = [
|
77
|
+
{ "message" => " foobar , hugo , abc def " },
|
78
|
+
]
|
79
|
+
expected = [
|
80
|
+
{ "message" => "foobar" },
|
81
|
+
{ "message" => "hugo" },
|
82
|
+
{ "message" => "abc def" },
|
83
|
+
]
|
84
|
+
filtered_records = filter(%[], messages)
|
85
|
+
assert_equal(expected, filtered_records)
|
86
|
+
end
|
87
|
+
end
|
metadata
ADDED
@@ -0,0 +1,114 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-filter-split-message
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Matteias Collet
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2023-10-03 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '2.1'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '2.1'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '13.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '13.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: test-unit
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3.3'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3.3'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: fluentd
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1'
|
69
|
+
description: Message splitter filter for Fluentd
|
70
|
+
email:
|
71
|
+
- matteias.collet@protonmail.ch
|
72
|
+
executables: []
|
73
|
+
extensions: []
|
74
|
+
extra_rdoc_files: []
|
75
|
+
files:
|
76
|
+
- ".devcontainer/Dockerfile"
|
77
|
+
- ".devcontainer/devcontainer.json"
|
78
|
+
- ".devcontainer/post-create.sh"
|
79
|
+
- ".github/workflows/build-and-test.yml"
|
80
|
+
- ".github/workflows/publish.yml"
|
81
|
+
- ".gitignore"
|
82
|
+
- ".vscode/settings.json"
|
83
|
+
- Gemfile
|
84
|
+
- LICENSE
|
85
|
+
- README.md
|
86
|
+
- Rakefile
|
87
|
+
- fluent-plugin-filter-split-message.gemspec
|
88
|
+
- lib/fluent/plugin/filter_split_message.rb
|
89
|
+
- test/test_split.rb
|
90
|
+
homepage: https://github.com/bitpatty/fluent-plugin-filter-split-message
|
91
|
+
licenses:
|
92
|
+
- MIT
|
93
|
+
metadata: {}
|
94
|
+
post_install_message:
|
95
|
+
rdoc_options: []
|
96
|
+
require_paths:
|
97
|
+
- lib
|
98
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - ">="
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '0'
|
103
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
104
|
+
requirements:
|
105
|
+
- - ">="
|
106
|
+
- !ruby/object:Gem::Version
|
107
|
+
version: '0'
|
108
|
+
requirements: []
|
109
|
+
rubygems_version: 3.3.5
|
110
|
+
signing_key:
|
111
|
+
specification_version: 4
|
112
|
+
summary: Message splitter filter for Fluentd
|
113
|
+
test_files:
|
114
|
+
- test/test_split.rb
|