fluent-plugin-filter-split-message 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: fd65ccd43f2d723119a6b1ac0285296396adf742b384fce09b682d36f3d2872e
+  data.tar.gz: e4954f1d238d4dfffc5251f79d04a515a64975d96e76a796644f910967d36db6
+SHA512:
+  metadata.gz: 15d4a54ce99d94390d977049eec2bdd84c2e6f79d381b04ad9ff920258e1b0714973df3f096f08004af60880d40c26efbff1adc8c06c1ff6d91c9838344e980d
+  data.tar.gz: d68bb390e637ac241c9bd77d0d29228047b8f88d3fe133b5aa34ce702a44d90bc7b343f08e8d7e1de3c2ee116965c5351233f490a8bd5980f6c962b4845d516d

data/.devcontainer/Dockerfile

@@ -0,0 +1,10 @@
+FROM ruby:3.0.3-bullseye
+
+RUN useradd -ms /bin/bash vscode \
+    && usermod -aG sudo vscode
+
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER vscode
+RUN gem install solargraph
+RUN gem install rufo

data/.devcontainer/devcontainer.json

@@ -0,0 +1,17 @@
+{
+	"name": "Fluent Plugin Filter Message Splitter",
+	"build": {
+		"dockerfile": "Dockerfile"
+	},
+	"customizations": {
+		"vscode":{
+			"extensions": [
+				"rebornix.Ruby",
+				"castwide.solargraph",
+				"jnbt.vscode-rufo"
+			]
+		}
+	},
+	"remoteUser": "vscode",
+	"postCreateCommand": "bash -i .devcontainer/post-create.sh"
+}

data/.devcontainer/post-create.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -e
+
+gem install bundler
+bundle install

data/.github/workflows/build-and-test.yml

@@ -0,0 +1,26 @@
+name: Build and Test
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.6', '2.7', '3.0' ]
+        os:
+          - ubuntu-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v3
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/.github/workflows/publish.yml

@@ -0,0 +1,23 @@
+name: Publish Gem
+
+on:
+  release:
+    types:
+      - created
+
+jobs:
+  publish:
+    if: github.repository_owner == 'bitpatty'
+    runs-on: ubuntu-latest
+    environment: package-registries
+    permissions:
+      contents: read
+      packages: write
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Publish gem
+      uses: dawidd6/action-publish-gem@v1
+      with:
+        api_key: ${{ secrets.RUBYGEMS_API_KEY }}
+        github_token: ${{ secrets.GITHUB_TOKEN }}

data/.gitignore

@@ -0,0 +1 @@
+*.lock

data/.vscode/settings.json

@@ -0,0 +1,21 @@
+{
+  "files.autoSave": "off",
+  "breadcrumbs.symbolSortOrder": "type",
+  "editor.codeLens": true,
+  "editor.detectIndentation": true,
+  "editor.formatOnSave": true,
+  "editor.minimap.maxColumn": 150,
+  "editor.tabSize": 2,
+  "explorer.confirmDragAndDrop": false,
+  "git.confirmSync": false,
+  "git.enableSmartCommit": true,
+  "html.format.wrapLineLength": 150,
+  "javascript.updateImportsOnFileMove.enabled": "always",
+  "typescript.preferences.importModuleSpecifier": "relative",
+  "typescript.referencesCodeLens.enabled": true,
+  "typescript.referencesCodeLens.showOnAllFunctions": true,
+  "typescript.reportStyleChecksAsWarnings": true,
+  "typescript.updateImportsOnFileMove.enabled": "always",
+  "workbench.editor.enablePreview": false,
+  "workbench.editor.enablePreviewFromQuickOpen": false
+}

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2017 Active Shadow LLC
+Copyright (c) 2023 Matteias Collet
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,45 @@
+# fluent-plugin-filter-split-message
+
+A [fluentd](https://www.fluentd.org/) filter plugin for splitting messages by a customizable delimiter.
+
+## Sample
+
+```ruby
+  {"message": "abc,def,ghi"}
+```
+
+.. is turned into ..
+
+```ruby
+  {"message": "abc"}
+  {"message": "def"}
+  {"message": "ghi"}
+```
+
+## Usage
+
+Install the plugin:
+
+```sh
+# See https://github.com/BitPatty/fluent-plugin-filter-split-message/releases for a list of valid versions
+gem install fluent-plugin-filter-split-message --version "<desired version>"
+```
+
+Create a logdrain and update your fluent configuration:
+
+```conf
+<filter>
+  @type split_message
+
+  # (Optional) The delimiter to use, defaults to ","
+  delimiter ,
+
+  # (Optional) The target field, defaults to "message"
+  field_key message
+</source>
+```
+
+
+## Credit
+
+- [fluent-plugin-split-event](https://github.com/uken/fluent-plugin-split-event) used as plugin reference

data/Rakefile

@@ -0,0 +1,13 @@
+require "bundler"
+Bundler::GemHelper.install_tasks
+
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs.push("lib", "test")
+  t.test_files = FileList["test/**/test_*.rb"]
+  t.verbose = true
+  t.warning = true
+end
+
+task default: [:test]

data/fluent-plugin-filter-split-message.gemspec

@@ -0,0 +1,28 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name = "fluent-plugin-filter-split-message"
+  spec.version = "0.1.0"
+  spec.authors = ["Matteias Collet"]
+  spec.email = ["matteias.collet@protonmail.ch"]
+
+  spec.summary = %q{Message splitter filter for Fluentd}
+  spec.description = spec.summary
+  spec.homepage = "https://github.com/bitpatty/fluent-plugin-filter-split-message"
+  spec.license = "MIT"
+
+  test_files, files = `git ls-files -z`.split("\x0").partition do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+
+  spec.files = files
+  spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files = test_files
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 2.1"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "test-unit", "~> 3.3"
+  spec.add_runtime_dependency "fluentd", ">= 1"
+end

data/lib/fluent/plugin/filter_split_message.rb

@@ -0,0 +1,47 @@
+require "fluent/plugin/filter"
+require "fluent/time"
+
+module Fluent
+  module Plugin
+    class SplitMessage < Filter
+      Plugin.register_filter("split_message", self)
+
+      config_param :field_key, :string, default: "message"
+      config_param :delimiter, :string, default: ","
+
+      def configure(conf)
+        super
+      end
+
+      def filter_stream(tag, es)
+        new_es = Fluent::MultiEventStream.new
+
+        es.each do |time, record|
+          begin
+            unless record.key?(@field_key)
+              return new_es
+            end
+
+            vals = record[@field_key].split(@delimiter)
+
+            if vals.count > 1
+              vals.each do |v|
+                begin
+                  new_record = record.dup
+                  new_record[@field_key] = v.strip
+                  new_es.add(time, new_record)
+                end
+              end
+            else
+              new_es.add(time, record)
+            end
+          rescue => e
+            router.emit_error_event(tag, time, record, e)
+          end
+        end
+
+        return new_es
+      end
+    end
+  end
+end

data/test/test_split.rb

@@ -0,0 +1,87 @@
+require "test/unit"
+require "fluent/test"
+require "fluent/test/driver/filter"
+require "fluent/plugin/filter_split_message.rb"
+
+class SplitMessageTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup # this is required to setup router and others
+  end
+
+  def create_driver(conf = "")
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::SplitMessage).configure(conf)
+  end
+
+  def filter(config, messages)
+    d = create_driver(config)
+    d.run(default_tag: "test") do
+      messages.each do |message|
+        d.feed(message)
+      end
+    end
+    d.filtered_records
+  end
+
+  test "empty message" do
+    messages = [
+      { "message" => "" },
+    ]
+    expected = [
+      { "message" => "" },
+    ]
+    filtered_records = filter("", messages)
+    assert_equal(expected, filtered_records)
+  end
+
+  test "simple split" do
+    messages = [
+      { "message" => "foobar,abc" },
+    ]
+    expected = [
+      { "message" => "foobar" },
+      { "message" => "abc" },
+    ]
+    filtered_records = filter("", messages)
+    assert_equal(expected, filtered_records)
+  end
+
+  test "multi char split" do
+    messages = [
+      { "message" => "foobar,,,,abc,,,," },
+    ]
+    expected = [
+      { "message" => "foobar" },
+      { "message" => "abc" },
+    ]
+    filtered_records = filter(%[
+      delimiter ',,,,'
+    ], messages)
+    assert_equal(expected, filtered_records)
+  end
+
+  test "multi split" do
+    messages = [
+      { "message" => "foobar,hugo,abc" },
+    ]
+    expected = [
+      { "message" => "foobar" },
+      { "message" => "hugo" },
+      { "message" => "abc" },
+    ]
+    filtered_records = filter(%[], messages)
+    assert_equal(expected, filtered_records)
+  end
+
+  test "trim" do
+    messages = [
+      { "message" => "    foobar    ,   hugo  ,  abc   def   " },
+    ]
+    expected = [
+      { "message" => "foobar" },
+      { "message" => "hugo" },
+      { "message" => "abc   def" },
+    ]
+    filtered_records = filter(%[], messages)
+    assert_equal(expected, filtered_records)
+  end
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-filter-split-message
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Matteias Collet
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2023-10-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1'
+description: Message splitter filter for Fluentd
+email:
+- matteias.collet@protonmail.ch
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".devcontainer/Dockerfile"
+- ".devcontainer/devcontainer.json"
+- ".devcontainer/post-create.sh"
+- ".github/workflows/build-and-test.yml"
+- ".github/workflows/publish.yml"
+- ".gitignore"
+- ".vscode/settings.json"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-filter-split-message.gemspec
+- lib/fluent/plugin/filter_split_message.rb
+- test/test_split.rb
+homepage: https://github.com/bitpatty/fluent-plugin-filter-split-message
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.5
+signing_key: 
+specification_version: 4
+summary: Message splitter filter for Fluentd
+test_files:
+- test/test_split.rb