fluent-plugin-formatter-protobuf 0.0.3 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5522a7376f600ab04c25874fed66cf1e3fa80488aff7917a0b4a20f2ae23e5a7
-  data.tar.gz: 56ff168034323c60b1c6482aa9be5d33540d6e1cdf58e8cb22f92cea5bfd8a6c
+  metadata.gz: ca1a8fea393af2eed56eac08ac8fe4f8122d0c1e5b5d36099580ff9e0c962c67
+  data.tar.gz: 9670c004adcabfe98e6ae98fe46a45ebe2a55e08df52d08e169fa149af2d032b
 SHA512:
-  metadata.gz: ef2e9c5998ed03fb9d24e5107185107cc37d9b5a803b1ab7fb2cedd624d55db04d62f32a55b4c80fb3422617f427ad48d168788b43891e6883ac908bbc9d90f5
-  data.tar.gz: 46e42467e1df6ec0c4c9fb45312141882da3a23c181cf408be0eb314edea2eff6708d89629210b41ff3f5d979070f083dd2018cddf45bd2e332d2b8b8d3eb428
+  metadata.gz: 8e8f996d0940c53666eae2cea2b7d9b47a6d57a15d6c2038c7d83accf592d89018e8c864731e5b574449cc7c5d2a9b4ea6f4153cbb9dbdd4de84cc3e5c8a57ec
+  data.tar.gz: 6b398f92c55ad9f8f6bc92b8da8947a887fa8bc3b3bd838c7973552e8aa12e353e04fcd76e13f9d314ace7429ced3c47b877b69f4d8b6270f7aefcfc0cb72758

data/.github/workflows/build-and-test.yml

@@ -0,0 +1,54 @@
+name: Build and test
+
+on:
+  push:
+
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.5
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+      - name: "Unit test"
+        run: "bundle exec rake lint:check"
+
+  test:
+    name: Unit test
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.5
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+      - name: "Unit test"
+        run: "bundle exec rake test:unit"
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.5
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+      - name: "Unit test"
+        run: "bundle exec rake build"

data/.github/workflows/codeql-analysis.yml

@@ -0,0 +1,49 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '18 11 * * 4'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'ruby' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

data/.github/workflows/publish.yml

@@ -0,0 +1,96 @@
+name: Publish
+
+on:
+  push:
+    tags:
+      - v0.*
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.5
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+      - name: "Unit test"
+        run: "bundle exec rake lint:check"
+
+  test:
+    name: Unit test
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.5
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+      - name: "Unit test"
+        run: "bundle exec rake test:unit"
+
+  publish-to-gpr:
+    needs:
+      - lint
+      - test
+    name: Build + Publish to GPR
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.5
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+
+      - name: Publish to GPR
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} *.gem
+        env:
+          GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
+          OWNER: ${{ github.repository_owner }}
+
+  publish-to-ruby-gems:
+    needs:
+      - lint
+      - test
+    name: Build + Publish to RubyGems
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.5
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+
+      - name: Publish to RubyGems
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          gem push *.gem
+        env:
+          GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"

data/.gitignore

@@ -1,2 +1,2 @@
-yarn-error.log
-node_modules
+*.gem
+Gemfile.lock

data/.rubocop.yml

@@ -1,13 +1,16 @@
+require: rubocop-rake
 AllCops:
   NewCops: enable
 Style/StringLiterals:
   Exclude:
-    - 'test/proto/addressbook_pb.rb'
+    - '**/*_pb.rb'
 Style/HashSyntax:
   Exclude:
-    - 'test/proto/addressbook_pb.rb'
+    - '**/*_pb.rb'
 Style/FrozenStringLiteralComment:
   Exclude:
-    - 'test/proto/addressbook_pb.rb'
+    - '**/*_pb.rb'
 Metrics/MethodLength:
-  Max: 100
+  Max: 100
+  Exclude:
+    - '**/*_pb.rb'

data/.ruby-version

@@ -1 +1 @@
-2.7.1
+2.5.0

data/README.md

@@ -1,21 +1,40 @@
 # fluent-plugin-formatter-protobuf
 
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-formatter-protobuf.svg)](https://badge.fury.io/rb/fluent-plugin-formatter-protobuf)
+
 [Fluentd](https://fluentd.org/) formatter plugin to format messages into Protobuf v3 encoded binary.
 
 ## Installation
 
 ### RubyGems
 
-```
+```shell
 $ gem install fluent-plugin-formatter-protobuf
 ```
 
-### Bundler
+### GitHub RubyGems Registry
+
+```shell
+$ gem install \
+  fluent-plugin-formatter-protobuf \
+  --version "<version>" \
+  --source "https://rubygems.pkg.github.com/fluent-plugins-nursery"
+```
+
+### Bundler (RubyGems)
 
 Add following line to your Gemfile:
 
 ```ruby
-gem "fluent-plugin-formatter-protobuf"
+gem "fluent-plugin-formatter-protobuf", "<version>"
+```
+
+### Bundler (GitHub RubyGems Registry),
+
+```shell
+source "https://rubygems.pkg.github.com/fluent-plugins-nursery" do
+  gem "fluent-plugin-formatter-protobuf", "<version>"
+end
 ```
 
 And then execute:
@@ -24,18 +43,98 @@ And then execute:
 $ bundle
 ```
 
+## Quick start
+
+1. Generate the protobuf ruby methods
+   1. e.g `protoc --proto_path=. --ruby_out=. ./log.proto`
+2. Modify the `<format>` section to include `class_name`, which is your Protobuf message name, and `include_paths`, the path where the generated Ruby types are stored
+   1. Given protobuf class `Your::Protobuf::Class::Name` class should be given as `Your.Protobuf.Class.Name` in `class_name`. The exact name can be found in the generated Ruby files
+
+## Note
+
+This plugin only supports Protobuf v3.
+
+## Example
+
+```aconf
+<source>
+    @type tcp
+    tag tcp.events
+    port 5170
+    delimiter "\n"
+    <parse>
+        @type json
+    </parse>
+</source>
+
+<match tcp.events>
+    @type kafka2
+    
+    brokers "#{BROKER_ENDPOINTS}"
+    compression_codec lz4
+
+    <buffer topic>
+        @type memory
+        flush_interval 5s
+    </buffer>
+
+    <format>
+        @type protobuf
+        class_name "Log"
+        include_paths ["/opt/fluent-plugin-formatter-protobuf/log_pb.rb"]
+    </format>
+</match>
+```
+
 ## Configuration
 
-You can generate configuration template:
+|parameter| type               | description                                                   | default |
+|---|--------------------|---------------------------------------------------------------|---------|
+|include_paths| array (optional)   | Generated Ruby Protobuf class files path                      | `[]`    |
+|class_name| string (required)  | Ruby Protobuf class name. Used to encode into Protobuf binary ||
+|decode_json| boolean (optional) | Serializes record from canonical proto3 JSON mapping (https://developers.google.com/protocol-buffers/docs/proto3#json) into binary                                                              | `false`  |
+
+## Tips
+
+#### 1. I have multiple `_pb.rb` files and they depend on one another. When I use this plugin, I'm getting a `LoadError`.
+
+This happens because the plugin currently loads paths in `include_paths` sequentially. You can either sort the files in correct dependency order (which is cumbersome), or add to Ruby's `$LOAD_PATH`.
+
+For example, you have 2 generated ruby files, `a_pb.rb` and `b_pb.rb`, and both `a_pb.rb` depends on `b_pb.rb`.
+
+You can either order them in appropriate dependency order, such as
+
+```aconf
+<format>
+  @type protobuf
+  class_name "SomePackage.A"
+  include_paths ["/some/path/b_pb.rb", "/some/path/a_pb.rb"]
+</format>
 
 ```
-$ fluent-plugin-config-format formatter formatter-protobuf
+
+or you can put the generated files in a directory say `/some/path/`, and add to Ruby's `$LOAD_PATH` via the `RUBYLIB` environment variable.
+
+e.g.
+
+```aconf
+<format>
+  @type protobuf
+  class_name "SomePackage.A"
+  include_paths ["/some/path/a_pb.rb"]
+</format>
+
 ```
 
-You can copy and paste generated documents here.
+```shell
+
+export RUBYLIB="${RUBYLIB}:/some/path/"
+fluentd \
+  --gemfile "/some/other/path/Gemfile" \
+  --config "/some/other/path/fluent.conf"
+
+```
 
-## Copyright
+## License
 
-* Copyright(c) 2021 - Ray Tung
-* License
-  * Apache License, Version 2.0
+[Apache License, Version 2.0](./LICENSE)

data/RELEASE.md

@@ -0,0 +1,9 @@
+# Release
+
+## Instructions
+To release the gem to Github package manager and Ruby Gems,
+
+1. Update `lib/fluent/plugin/version.rb`
+2. Create a pull request and merge it into `main`
+3. Manually creating a release on Github with a tag indicating the new version number e.g. `vMAJOR.MINOR.PATCH`
+4. GitHub Actions will automatically run and publish to RubyGems and GitHub repository

data/Rakefile

@@ -44,6 +44,11 @@ namespace :lint do
   end
 end
 
+desc 'Building ruby gem'
+task :build do
+  sh 'gem build *.gemspec'
+end
+
 desc 'Listing all tasks'
 task :help do
   sh 'rake --tasks'

data/fluent-plugin-formatter-protobuf.gemspec

@@ -6,15 +6,16 @@ require 'fluent/plugin/version'
 
 Gem::Specification.new do |spec|
   spec.name = 'fluent-plugin-formatter-protobuf'
-  spec.version = Fluent::Plugin::ProtobufFormatter::VERSION
+  spec.version = Fluent::Plugin::VERSION
   spec.authors = ['Ray Tung']
   spec.email = ['code@raytung.net']
+  spec.metadata['rubygems_mfa_required'] = 'true'
 
   spec.summary = 'Protobuf formatter for Fluentd'
   spec.description = 'This is a Fluentd formatter plugin designed to convert Protobuf JSON into Protobuf binary'
   spec.homepage = 'https://github.com/raytung/fluent-plugin-formatter-protobuf'
   spec.license = 'Apache-2.0'
-  spec.required_ruby_version = Gem::Requirement.new('>=2.7.1')
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.5.0')
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = 'https://github.com/raytung/fluent-plugin-formatter-protobuf'
@@ -28,9 +29,12 @@ Gem::Specification.new do |spec|
   spec.test_files = test_files
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '~> 2.2.22'
-  spec.add_development_dependency 'rake', '~> 13.0.3'
-  spec.add_development_dependency 'test-unit', '~> 3.3.7'
+  spec.add_development_dependency 'bundler', '~> 2.2'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rubocop', '~> 1.22'
+  spec.add_development_dependency 'rubocop-rake', '~> 0.6'
+  spec.add_development_dependency 'test-unit', '~> 3.3'
   spec.add_runtime_dependency 'fluentd', ['>= 1.0', '< 2']
   spec.add_runtime_dependency 'google-protobuf', ['~> 3.18']
+  spec.add_runtime_dependency 'oj', '~> 3.13'
 end

data/lib/fluent/plugin/formatter_protobuf.rb

@@ -22,6 +22,7 @@ require 'google/protobuf'
 
 require 'fluent/env'
 require 'fluent/time'
+require 'oj'
 
 module Fluent
   module Plugin
@@ -29,11 +30,12 @@ module Fluent
     class ProtobufFormatter < Fluent::Plugin::Formatter
       Fluent::Plugin.register_formatter('protobuf', self)
 
-      # Absolute paths to the generated Ruby protobuf files
-      config_param :include_paths, :array, default: []
+      config_param :include_paths, :array, default: [], desc: 'Generated Ruby Protobuf class files path'
 
-      # Protobuf message name
-      config_param :message_name, :string
+      config_param :class_name, :string, desc: 'Ruby Protobuf class name. Used to encode into Protobuf binary'
+
+      config_param :decode_json, :bool, default: false,
+                                        desc: 'Serializes record from canonical proto3 JSON mapping (https://developers.google.com/protocol-buffers/docs/proto3#json) into binary' # rubocop:disable Layout/LineLength
 
       def configure(conf)
         super(conf)
@@ -42,10 +44,10 @@ module Fluent
 
         @include_paths.each { |path| require_proto!(path) } unless @include_paths.empty?
 
-        message_lookup = Google::Protobuf::DescriptorPool.generated_pool.lookup(@message_name)
-        raise Fluent::ConfigError, "message name '#{@message_name}' not found" if message_lookup.nil?
+        class_lookup = Google::Protobuf::DescriptorPool.generated_pool.lookup(@class_name)
+        raise Fluent::ConfigError, "class name '#{@class_name}' not found" if class_lookup.nil?
 
-        @protobuf_class = message_lookup.msgclass
+        @protobuf_class = class_lookup.msgclass
       end
 
       def formatter_type
@@ -53,26 +55,18 @@ module Fluent
       end
 
       def format(_tag, _time, record)
-        protobuf_msg = @protobuf_class.new(record)
+        protobuf_msg = @decode_json ? @protobuf_class.decode_json(Oj.dump(record)) : @protobuf_class.new(record)
         @protobuf_class.encode(protobuf_msg)
       end
 
-      # @param [string] filename
-      # @return [void]
       def require_proto!(filename)
-        unless filename.end_with?('.rb')
-          raise Fluent::ConfigError, "Unable to load file '#{filename}'. It is not a Ruby file"
-        end
-
-        unless Pathname.new(filename).absolute?
-          raise Fluent::ConfigError, "Unable to load file '#{filename}'. Please provide absolute paths"
-        end
-
-        begin
+        if Pathname.new(filename).absolute?
           require filename
-        rescue LoadError => e
-          raise Fluent::ConfigError, "Unable to load file '#{filename}'. Reason: #{e.inspect}"
+        else
+          require_relative filename
         end
+      rescue LoadError => e
+        raise Fluent::ConfigError, "Unable to load file '#{filename}'. Reason: #{e.inspect}"
       end
     end
   end

data/lib/fluent/plugin/version.rb

@@ -2,8 +2,6 @@
 
 module Fluent
   module Plugin
-    module ProtobufFormatter
-      VERSION = '0.0.3'
-    end
+    VERSION = '0.1.1'
   end
 end

data/test/integration/Gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gem 'fluentd', '1.14.2'
+gem 'fluent-plugin-formatter-protobuf', '0.1.0'

data/{Gemfile.lock → test/integration/Gemfile.lock}

@@ -1,53 +1,45 @@
-PATH
-  remote: .
-  specs:
-    fluent-plugin-formatter-protobuf (0.0.3)
-      fluentd (>= 1.0, < 2)
-      google-protobuf (~> 3.18)
-
 GEM
   remote: https://rubygems.org/
   specs:
     concurrent-ruby (1.1.9)
     cool.io (1.7.1)
-    fluentd (1.14.1)
+    fluent-plugin-formatter-protobuf (0.1.0)
+      fluentd (>= 1.0, < 2)
+      google-protobuf (~> 3.18)
+    fluentd (1.14.2)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.8.0)
       msgpack (>= 1.3.1, < 2.0.0)
       serverengine (>= 2.2.2, < 3.0.0)
       sigdump (~> 0.2.2)
-      strptime (>= 0.2.2, < 1.0.0)
+      strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       webrick (>= 1.4.2, < 1.8.0)
       yajl-ruby (~> 1.0)
-    google-protobuf (3.18.1-x86_64-linux)
+    google-protobuf (3.19.1)
+    google-protobuf (3.19.1-x86_64-linux)
     http_parser.rb (0.7.0)
     msgpack (1.4.2)
-    power_assert (2.0.1)
-    rake (13.0.6)
     serverengine (2.2.4)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     strptime (0.2.5)
-    test-unit (3.3.9)
-      power_assert
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2021.3)
+    tzinfo-data (1.2021.5)
       tzinfo (>= 1.0.0)
     webrick (1.7.0)
     yajl-ruby (1.4.1)
 
 PLATFORMS
+  ruby
   x86_64-linux
 
 DEPENDENCIES
-  bundler (~> 2.2.22)
-  fluent-plugin-formatter-protobuf!
-  rake (~> 13.0.3)
-  test-unit (~> 3.3.7)
+  fluent-plugin-formatter-protobuf (= 0.1.0)
+  fluentd (= 1.14.2)
 
 BUNDLED WITH
    2.2.29

data/test/integration/README.md

@@ -0,0 +1,12 @@
+# Integration test
+
+A _manual_ integration test.
+
+### Instructions
+
+1. Open a new terminal and run`$ ./run.sh`
+2. Open a separate terminal and run `echo '{"timestamp": 1634386250, "message":"hello!"}\n' | netcat 127.0.0.1 5170`
+3. Somehow flush data (I simply stop the docker container)
+4. Verify out data (It should look like `out.YYYYMMDD_N.log`) with `protoc`
+   1. `protoc --decode_raw < out.YYYYMMDD_N.log`
+   2. `cat out.YYYYMMDD_N.log | protoc --decode=Log --proto_path=. log.proto`

data/test/integration/fluent.conf

@@ -0,0 +1,25 @@
+<source>
+    @type tcp
+    tag tcp.events
+    port 5170
+    delimiter "\n"
+    <parse>
+        @type json
+    </parse>
+</source>
+
+<match tcp.events>
+    @type file
+
+    path /opt/fluent-plugin-formatter-protobuf/out
+
+    <buffer>
+        @type memory
+    </buffer>
+
+    <format>
+        @type protobuf
+        class_name "Log"
+        include_paths ["/opt/fluent-plugin-formatter-protobuf/log_pb.rb"]
+    </format>
+</match>

data/test/integration/log.proto

@@ -0,0 +1,6 @@
+syntax = "proto3";
+
+message Log {
+  int64 timestamp = 1;
+  string message = 2;
+}

data/test/integration/log_pb.rb

@@ -0,0 +1,15 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: log.proto
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_file("log.proto", :syntax => :proto3) do
+    add_message "Log" do
+      optional :timestamp, :int64, 1
+      optional :message, :string, 2
+    end
+  end
+end
+
+Log = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("Log").msgclass