fluent-plugin-formatter-protobuf 0.0.1 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3d8d0f38e949a78909ca9396248df0f460dc3ac8901f66b5fa2f438da5284b0c
-  data.tar.gz: d74cb1a9098d00115b0b4bbe64355f853e3aa2d89b77a888014aabc70d22e2d8
+  metadata.gz: 9bcc9a5c80a25b6d30ebbabcbc6596db405738915dad875eb52570df1e793a0b
+  data.tar.gz: c05073cdb7b669ca81aa29b21fc46d2c50846072b38d3cd749f09666babc62f8
 SHA512:
-  metadata.gz: 4fdd5e38190985d2c1b10959b4ff5bd9729b82a4877647ee3b2d4e7c8bf0e05657686c560810cf6398bf256a537caccdc4ab9c5bf118dc1bfd6f0684d5483a8f
-  data.tar.gz: 34d8144ccb7a304eacd70034ad9e57e7b317d2eca459bb07bfe5295aa925baad00f45d78e89f385210450ac6cb2a41e443172ae697df6fdc29ec271a06062203
+  metadata.gz: efd71e67d8f9388d487d2d6de036c012a29847960f8eab02c39d8c6d08e7b6a20fb4a1c7d622b8fbdb1a06d67666d9a2885edcef2bbb769fc7f3278400102d46
+  data.tar.gz: 79a3e80a85c12df566a4e59b764487e9cb7e4d66962b0fcb0bbda6f8e840e8c64cc58a407ab948e256e19bf62aea46e0a9fe481b1c445966e9f3c6c2fed39ffe

data/.github/workflows/build-and-test.yml

@@ -0,0 +1,54 @@
+name: Build and test
+
+on:
+  push:
+
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.5
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+      - name: "Unit test"
+        run: "bundle exec rake lint:check"
+
+  test:
+    name: Unit test
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.5
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+      - name: "Unit test"
+        run: "bundle exec rake test:unit"
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.5
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+      - name: "Unit test"
+        run: "bundle exec rake build"

data/.github/workflows/publish.yml

@@ -0,0 +1,96 @@
+name: Publish
+
+on:
+  push:
+    tags:
+      - v0.*
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.5
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+      - name: "Unit test"
+        run: "bundle exec rake lint:check"
+
+  test:
+    name: Unit test
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.5
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+      - name: "Unit test"
+        run: "bundle exec rake test:unit"
+
+  publish-to-gpr:
+    needs:
+      - lint
+      - test
+    name: Build + Publish to GPR
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.5
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+
+      - name: Publish to GPR
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} *.gem
+        env:
+          GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
+          OWNER: ${{ github.repository_owner }}
+
+  publish-to-ruby-gems:
+    needs:
+      - lint
+      - test
+    name: Build + Publish to RubyGems
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.5
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+
+      - name: Publish to RubyGems
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          gem push *.gem
+        env:
+          GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"

data/.gitignore

@@ -1,2 +1,2 @@
-yarn-error.log
-node_modules
+*.gem
+Gemfile.lock

data/.rubocop.yml

@@ -1,13 +1,16 @@
+require: rubocop-rake
 AllCops:
   NewCops: enable
 Style/StringLiterals:
   Exclude:
-    - 'test/proto/addressbook_pb.rb'
+    - '**/*_pb.rb'
 Style/HashSyntax:
   Exclude:
-    - 'test/proto/addressbook_pb.rb'
+    - '**/*_pb.rb'
 Style/FrozenStringLiteralComment:
   Exclude:
-    - 'test/proto/addressbook_pb.rb'
+    - '**/*_pb.rb'
 Metrics/MethodLength:
-  Max: 100
+  Max: 100
+  Exclude:
+    - '**/*_pb.rb'

data/.ruby-version

@@ -1 +1 @@
-2.7.1
+2.5.0

data/README.md

@@ -1,21 +1,40 @@
 # fluent-plugin-formatter-protobuf
 
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-formatter-protobuf.svg)](https://badge.fury.io/rb/fluent-plugin-formatter-protobuf)
+
 [Fluentd](https://fluentd.org/) formatter plugin to format messages into Protobuf v3 encoded binary.
 
 ## Installation
 
 ### RubyGems
 
-```
+```shell
 $ gem install fluent-plugin-formatter-protobuf
 ```
 
-### Bundler
+### GitHub RubyGems Registry
+
+```shell
+$ gem install \
+  fluent-plugin-formatter-protobuf \
+  --version "<version>" \
+  --source "https://rubygems.pkg.github.com/raytung"
+```
+
+### Bundler (RubyGems)
 
 Add following line to your Gemfile:
 
 ```ruby
-gem "fluent-plugin-formatter-protobuf"
+gem "fluent-plugin-formatter-protobuf", "<version>"
+```
+
+### Bundler (GitHub RubyGems Registry),
+
+```shell
+source "https://rubygems.pkg.github.com/raytung" do
+  gem "fluent-plugin-formatter-protobuf", "<version>"
+end
 ```
 
 And then execute:
@@ -24,6 +43,43 @@ And then execute:
 $ bundle
 ```
 
+## Not so quick start
+
+1. Generate the protobuf ruby methods
+   1. e.g `protoc --proto_path=. --ruby_out=. ./log.proto`
+2. Modify the `<format>` section to include `message_name`, which is your Protobuf message name, and `include_paths`, the path where the generated Ruby types are stored
+
+
+## Example
+
+```fluentd
+<source>
+    @type tcp
+    tag tcp.events
+    port 5170
+    delimiter "\n"
+    <parse>
+        @type json
+    </parse>
+</source>
+
+<match tcp.events>
+    @type file
+
+    path /opt/fluent-plugin-formatter-protobuf/out
+
+    <buffer>
+        @type memory
+    </buffer>
+
+    <format>
+        @type protobuf
+        message_name "Log"
+        include_paths ["/opt/fluent-plugin-formatter-protobuf/log_pb.rb"]
+    </format>
+</match>
+```
+
 ## Configuration
 
 You can generate configuration template:

data/RELEASE.md

@@ -0,0 +1,9 @@
+# Release
+
+## Instructions
+To release the gem to Github package manager and Ruby Gems,
+
+1. Update `lib/fluent/version.rb`
+2. Update `Gemfile.lock`
+3. Create a pull request and merge it into `main`
+4. Manually creating a release on Github with a tag indicating the new version number e.g. `vMAJOR.MINOR.PATCH`

data/Rakefile

@@ -44,6 +44,11 @@ namespace :lint do
   end
 end
 
+desc 'Building ruby gem'
+task :build do
+  sh 'gem build *.gemspec'
+end
+
 desc 'Listing all tasks'
 task :help do
   sh 'rake --tasks'

data/fluent-plugin-formatter-protobuf.gemspec

@@ -6,7 +6,7 @@ require 'fluent/plugin/version'
 
 Gem::Specification.new do |spec|
   spec.name = 'fluent-plugin-formatter-protobuf'
-  spec.version = Fluent::Plugin::ProtobufFormatter::VERSION
+  spec.version = Fluent::Plugin::VERSION
   spec.authors = ['Ray Tung']
   spec.email = ['code@raytung.net']
 
@@ -14,11 +14,11 @@ Gem::Specification.new do |spec|
   spec.description = 'This is a Fluentd formatter plugin designed to convert Protobuf JSON into Protobuf binary'
   spec.homepage = 'https://github.com/raytung/fluent-plugin-formatter-protobuf'
   spec.license = 'Apache-2.0'
-  spec.required_ruby_version = Gem::Requirement.new('>=2.7.1')
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.5.0')
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = 'https://github.com/raytung/fluent-plugin-formatter-protobuf'
-  spec.metadata['changelog_uri'] = 'https://github.com/raytung/fluent-plugin-formatter-protobuf/blobl/main/CHANGELOG.md'
+  spec.metadata['changelog_uri'] = 'https://github.com/raytung/fluent-plugin-formatter-protobuf/blob/main/CHANGELOG.md'
 
   test_files, files = `git ls-files -z`.split("\x0").partition do |f|
     f.match(%r{^(test|spec|features)/})
@@ -30,6 +30,8 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 2.2.22'
   spec.add_development_dependency 'rake', '~> 13.0.3'
+  spec.add_development_dependency 'rubocop', '~> 1.22.1'
+  spec.add_development_dependency 'rubocop-rake', '~> 0.6.0'
   spec.add_development_dependency 'test-unit', '~> 3.3.7'
   spec.add_runtime_dependency 'fluentd', ['>= 1.0', '< 2']
   spec.add_runtime_dependency 'google-protobuf', ['~> 3.18']

data/lib/fluent/plugin/formatter_protobuf.rb

@@ -19,7 +19,6 @@ require 'fluent/plugin/formatter'
 
 require 'pathname'
 require 'google/protobuf'
-require 'protocol_buffers'
 
 require 'fluent/env'
 require 'fluent/time'
@@ -49,6 +48,10 @@ module Fluent
         @protobuf_class = message_lookup.msgclass
       end
 
+      def formatter_type
+        :binary
+      end
+
       def format(_tag, _time, record)
         protobuf_msg = @protobuf_class.new(record)
         @protobuf_class.encode(protobuf_msg)

data/lib/fluent/plugin/version.rb

@@ -2,8 +2,6 @@
 
 module Fluent
   module Plugin
-    module ProtobufFormatter
-      VERSION = '0.0.1'
-    end
+    VERSION = '0.0.5'
   end
 end

data/test/integration/Gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gem 'fluentd', '1.14.1'
+gem 'fluent-plugin-formatter-protobuf', '0.0.3'

data/{Gemfile.lock → test/integration/Gemfile.lock}

@@ -1,15 +1,11 @@
-PATH
-  remote: .
-  specs:
-    fluent-plugin-formatter-protobuf (0.0.1)
-      fluentd (>= 1.0, < 2)
-      google-protobuf (~> 3.18)
-
 GEM
   remote: https://rubygems.org/
   specs:
     concurrent-ruby (1.1.9)
     cool.io (1.7.1)
+    fluent-plugin-formatter-protobuf (0.0.3)
+      fluentd (>= 1.0, < 2)
+      google-protobuf (~> 3.18)
     fluentd (1.14.1)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
@@ -22,17 +18,14 @@ GEM
       tzinfo-data (~> 1.0)
       webrick (>= 1.4.2, < 1.8.0)
       yajl-ruby (~> 1.0)
+    google-protobuf (3.18.1)
     google-protobuf (3.18.1-x86_64-linux)
     http_parser.rb (0.7.0)
     msgpack (1.4.2)
-    power_assert (2.0.1)
-    rake (13.0.6)
     serverengine (2.2.4)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     strptime (0.2.5)
-    test-unit (3.3.9)
-      power_assert
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
     tzinfo-data (1.2021.3)
@@ -41,13 +34,12 @@ GEM
     yajl-ruby (1.4.1)
 
 PLATFORMS
+  ruby
   x86_64-linux
 
 DEPENDENCIES
-  bundler (~> 2.2.22)
-  fluent-plugin-formatter-protobuf!
-  rake (~> 13.0.3)
-  test-unit (~> 3.3.7)
+  fluent-plugin-formatter-protobuf (= 0.0.3)
+  fluentd (= 1.14.1)
 
 BUNDLED WITH
    2.2.29

data/test/integration/README.md

@@ -0,0 +1,12 @@
+# Integration test
+
+A _manual_ integration test.
+
+### Instructions
+
+1. Open a new terminal and run`$ ./run.sh`
+2. Open a separate terminal and run `echo '{"timestamp": 1634386250, "message":"hello!"}\n' | netcat 127.0.0.1 5170`
+3. Somehow flush data (I simply stop the docker container)
+4. Verify out data (It should look like `out.YYYYMMDD_N.log`) with `protoc`
+   1. `protoc --decode_raw < out.YYYYMMDD_N.log`
+   2. `cat out.YYYYMMDD_N.log | protoc --decode=Log --proto_path=. log.proto`

data/test/integration/fluent.conf

@@ -0,0 +1,25 @@
+<source>
+    @type tcp
+    tag tcp.events
+    port 5170
+    delimiter "\n"
+    <parse>
+        @type json
+    </parse>
+</source>
+
+<match tcp.events>
+    @type file
+
+    path /opt/fluent-plugin-formatter-protobuf/out
+
+    <buffer>
+        @type memory
+    </buffer>
+
+    <format>
+        @type protobuf
+        message_name "Log"
+        include_paths ["/opt/fluent-plugin-formatter-protobuf/log_pb.rb"]
+    </format>
+</match>

data/test/integration/log.proto

@@ -0,0 +1,6 @@
+syntax = "proto3";
+
+message Log {
+  int64 timestamp = 1;
+  string message = 2;
+}

data/test/integration/log_pb.rb

@@ -0,0 +1,15 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: log.proto
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_file("log.proto", :syntax => :proto3) do
+    add_message "Log" do
+      optional :timestamp, :int64, 1
+      optional :message, :string, 2
+    end
+  end
+end
+
+Log = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("Log").msgclass

data/test/integration/run.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+docker run -it --rm \
+  -v "${PWD}:/opt/fluent-plugin-formatter-protobuf/" \
+  -w '/opt/fluent-plugin-formatter-protobuf/' \
+  -e OUT_FILE="./out" \
+  -p 5170:5170 \
+  ruby:2.7.1 sh -c 'bundle install && fluentd -c /opt/fluent-plugin-formatter-protobuf/fluent.conf'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-formatter-protobuf
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.5
 platform: ruby
 authors:
 - Ray Tung
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-16 00:00:00.000000000 Z
+date: 2021-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,6 +38,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 13.0.3
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.22.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.22.1
+- !ruby/object:Gem::Dependency
+  name: rubocop-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -94,36 +122,42 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/workflows/gem-push.yml"
+- ".github/workflows/build-and-test.yml"
+- ".github/workflows/publish.yml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".ruby-version"
 - Gemfile
-- Gemfile.lock
 - LICENSE
 - README.md
+- RELEASE.md
 - Rakefile
 - docker-compose.yaml
 - docker-run.sh
 - fluent-plugin-formatter-protobuf.gemspec
 - lib/fluent/plugin/formatter_protobuf.rb
 - lib/fluent/plugin/version.rb
-- package.json
 - test/helper.rb
+- test/integration/Gemfile
+- test/integration/Gemfile.lock
+- test/integration/README.md
+- test/integration/fluent.conf
+- test/integration/log.proto
+- test/integration/log_pb.rb
+- test/integration/run.sh
 - test/plugin/test_formatter_protobuf.rb
 - test/proto/README.md
 - test/proto/addressbook
 - test/proto/addressbook.bin
 - test/proto/addressbook.proto
 - test/proto/addressbook_pb.rb
-- yarn.lock
 homepage: https://github.com/raytung/fluent-plugin-formatter-protobuf
 licenses:
 - Apache-2.0
 metadata:
   homepage_uri: https://github.com/raytung/fluent-plugin-formatter-protobuf
   source_code_uri: https://github.com/raytung/fluent-plugin-formatter-protobuf
-  changelog_uri: https://github.com/raytung/fluent-plugin-formatter-protobuf/blobl/main/CHANGELOG.md
+  changelog_uri: https://github.com/raytung/fluent-plugin-formatter-protobuf/blob/main/CHANGELOG.md
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -132,19 +166,27 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.1
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubyforge_project: 
+rubygems_version: 2.7.3
 signing_key: 
 specification_version: 4
 summary: Protobuf formatter for Fluentd
 test_files:
 - test/helper.rb
+- test/integration/Gemfile
+- test/integration/Gemfile.lock
+- test/integration/README.md
+- test/integration/fluent.conf
+- test/integration/log.proto
+- test/integration/log_pb.rb
+- test/integration/run.sh
 - test/plugin/test_formatter_protobuf.rb
 - test/proto/README.md
 - test/proto/addressbook

data/.github/workflows/gem-push.yml

@@ -1,19 +0,0 @@
-name: Publish Package
-on:
-  push:
-    branches:
-      - "main"
-  pull_request:
-    branches:
-      - "main"
-jobs:
-  release:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2.3.5
-      - uses: actions/setup-node@v2.4.1
-        with:
-          node-version: "16"
-      - uses: ruby/setup-ruby@v1.83.0
-        with:
-          ruby-version: "2.7.1"