fluent-plugin-protobuf-http 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce7ced8f827b15e5e9520ccedf03984720014de458c653ce55a2cb3a8b2c0b4d
-  data.tar.gz: aec3d4403856f8d5f67f55c143d5e835b0d027dc8ef0599d952d06544a738795
+  metadata.gz: c12d1da7ec8d1e801e5c9bc01ae3258e4c13ee6d361431721305e8a3778fe0d5
+  data.tar.gz: 2571d4e0c3795fd37c87d07fcbfd80fc7179b7bab0c7d6760c07b3fc7a561e73
 SHA512:
-  metadata.gz: db3abc4b52dd4d9c69bc8b8f493877ce65308e2d4deca4bce4c0888d2b3e1e6d1ee39fb8c0f0ccfb43e0ef5a3367e7883feb4ecb4d104effb5ad26dd549adc3a
-  data.tar.gz: 36bd9d1a8877f9cd31ae66eacde63b99d89fa6d6adc29e4ed6f5240eaf713a05b1c8b8828945eac84a062fc4e1081030e00b6924eded3e55b8e287edaa0f55da
+  metadata.gz: cfb5aba44bec8f2a9ef10514ef50f5fc2d14084ccd16a8b8bb8d25babc82f351b9bed58ee3c290e00af11fb6dbcb06f4c96d41c342ee67c33fd8b42f6db1399c
+  data.tar.gz: e5e01488511a1509becf3f636e1ab0e653e243dbb44516279555963ed03b09d3333a92d9e8af07a9b9494f75b285adb97e8c7b3ac1f270111b50359054702f80

data/.github/workflows/ci.yml

@@ -0,0 +1,48 @@
+name: ci
+
+on:
+  push:
+    paths-ignore:
+      - '**.md'
+      - '.rubocop.yml'
+  pull_request:
+    paths-ignore:
+      - '**.md'
+      - '.rubocop.yml'
+
+jobs:
+  run-tests:
+    name: Run tests
+    strategy:
+      matrix:
+        os: [ubuntu-18.04]
+        ruby-version: ['2.5', '2.6', '2.7']
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true
+
+    - name: Bundle install
+      run: |
+        gem install bundler && \
+        bundle install --jobs 4 --retry 3
+
+    - name: Install protoc
+      run: |
+        mkdir protoc && cd protoc
+        wget https://github.com/protocolbuffers/protobuf/releases/download/v3.17.0/protoc-3.17.0-linux-x86_64.zip
+        unzip protoc-3.17.0-linux-x86_64.zip
+        chmod +x ./bin/protoc && sudo mv ./bin/protoc /usr/local/bin/protoc
+        sudo mv ./include/* /usr/local/include/
+        cd .. && rm -rf ./protoc
+        protoc --version
+
+    - name: Tests
+      run: bundle exec rake test

data/.rubocop.yml

@@ -0,0 +1,19 @@
+# RuboCop Default Config: https://github.com/rubocop-hq/rubocop/blob/master/config/default.yml
+
+AllCops:
+  NewCops: disable
+
+Gemspec/RequiredRubyVersion:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false
+
+Layout/LineLength:
+  Enabled: false

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen-string-literal: true
+
 source 'https://rubygems.org'
 
 gemspec

data/README.md

@@ -1,62 +1,79 @@
 # fluent-plugin-protobuf-http
 
+[![ci](https://github.com/iamazeem/fluent-plugin-protobuf-http/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/iamazeem/fluent-plugin-protobuf-http/actions/workflows/ci.yml)
+[![License: Apache](https://img.shields.io/badge/license-Apache-blue.svg?style=flat-square)](https://github.com/iamAzeem/fluent-plugin-protobuf-http/blob/master/LICENSE)
+![GitHub release (latest by date)](https://img.shields.io/github/v/release/iamAzeem/fluent-plugin-protobuf-http?style=flat-square)
+[![RubyGems Downloads](https://img.shields.io/gem/dt/fluent-plugin-protobuf-http?color=blue&style=flat-square)](https://rubygems.org/gems/fluent-plugin-protobuf-http)
+
+![Lines of code](https://img.shields.io/tokei/lines/github/iamAzeem/fluent-plugin-protobuf-http?label=LOC&style=flat-square)
+![GitHub code size in bytes](https://img.shields.io/github/languages/code-size/iamAzeem/fluent-plugin-protobuf-http?style=flat-square)
+![GitHub repo size](https://img.shields.io/github/repo-size/iamAzeem/fluent-plugin-protobuf-http?style=flat-square)
+
 [Fluentd](https://fluentd.org/) HTTP input plugin for Protocol Buffers.
 
 ## Features
 
-* **ProtoBuf Schemas**: Automatic compilation of `.proto` files located in `proto_dir`
-* **Incoming Message Format**: Support for binary or JSON format (`Content-Type`: `application/octet-stream` or `application/json`)
-* **Outgoing Message Format**: Support for binary or JSON format
-* **Message Types**: Single or Batch
-* **TLS Support**: Use `<transport tls> ... </transport>` section in configuration and `https://` URL protocol prefix. See this [example](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-server#configuration-example) for more details.
+* Automatic compilation of `.proto` files located in `proto_dir`
+* Incoming Format: Binary or JSON (`Content-Type`: `application/octet-stream` or
+  `application/json`)
+* Outgoing Format: Binary or JSON
+* Single and Batch message support
+* TLS Support with `<transport>` section and `https://` URL protocol prefix.
+
+For more details on TLS configuration, see this official
+[example](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-server#configuration-example).
 
 ## Installation
 
 ### RubyGems
 
-```
-$ gem install fluent-plugin-protobuf-http
+```shell
+gem install fluent-plugin-protobuf-http
 ```
 
 ### Bundler
 
 Add following line to your Gemfile:
+
 ```ruby
-gem "fluent-plugin-protobuf-http"
+gem 'fluent-plugin-protobuf-http'
 ```
 
 And then execute:
-```
-$ bundle
+
+```shell
+bundle
 ```
 
 ## Configuration
 
-* **bind** (string) (optional): The address to listen to.
-  * Default value: `0.0.0.0`.
-* **port** (integer) (optional): The port to listen to.
-  * Default value: `8080`.
-* **proto_dir** (string) (required): The directory path that contains the .proto files.
-* **in_mode** (enum) (optional): The mode of incoming (supported) events.
-  * Available values: binary, json
-  * Default value: `binary`.
-* **out_mode** (enum) (optional): The mode of outgoing (emitted) events.
-  * Available values: binary, json
-  * Default value: `binary`.
-* **tag** (string) (required): The tag for the event.
-
-### \<transport\> section (optional) (single)
-
-* **protocol** (enum) (optional):
-  * Available values: tcp, tls
-  * Default value: `tcp`.
-  * See [example](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-server#configuration-example).
+* `bind` (string) (optional): The address to listen to.
+  * Default: `0.0.0.0`
+* `port` (integer) (optional): The port to listen to.
+  * Default: `8080`
+* `proto_dir` (string) (required): The directory path that contains the .proto files.
+* `in_mode` (enum) (optional): The mode of incoming (supported) events.
+  * Modes: `binary`, `json`
+  * Default: `binary`
+* `out_mode` (enum) (optional): The mode of outgoing (emitted) events.
+  * Modes: `binary`, `json`
+  * Default: `binary`
+* `tag` (string) (required): The tag for the event.
+
+### `<transport>` section (optional) (single)
+
+* `protocol` (enum) (optional):
+  * Protocols: `tcp`, `tls`
+  * Default: `tcp`
+  * For more details, see this official configuration
+    [example](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-server#configuration-example).
 
 ### Example
 
-```
-# Single Message: http://ip:port/<tag>?msgtype=<msgtype>
-# Batch  Message: http://ip:port/<tag>?msgtype=<batch-msgtype>?batch=true
+```text
+# Endpoints:
+# - Single Message: http://ip:port/<tag>?msgtype=<msgtype>
+# - Batch  Message: http://ip:port/<tag>?msgtype=<batch-msgtype>?batch=true
 
 <source>
   @type       protobuf_http
@@ -74,12 +91,18 @@ $ bundle
 
 ## Schemas (`.proto` files)
 
-The logging of events is assumed to be the prime use-case for this plugin.  
-So, use self-contained `.proto` file(s) that don't import other custom `.proto` file(s).  
-The `package` and `message` names must be unique and are treated as case-sensitive.
+The prime use-case for this plugin is assumed to be event logging. So, always
+use self-contained `.proto` file(s) that do not import other `.proto` files. The
+names e.g. `package`, `message`, etc. must be unique and are treated as
+case-sensitive.
 
-Consider this [`log.proto`](https://github.com/iamAzeem/protobuf-log-sample/blob/master/log.proto) schema:
-```
+Consider this
+[log.proto](https://github.com/iamAzeem/protobuf-log-sample/blob/master/log.proto)
+schema from
+[protobuf-log-sample](https://github.com/iamAzeem/protobuf-log-sample)
+repository:
+
+```protobuf
 syntax = "proto3";
 
 package service.logging;
@@ -108,9 +131,9 @@ message Log {
 }
 ```
 
-The fully-qualified message type for `Log` will be `service.logging.Log`.  
-This message type is used as the value of `msgtype` query parameter in the URL.  
-See URL section below for more on `msgtype`.
+The fully-qualified message type for `Log` is `service.logging.Log`. This
+message type is used as the value of `msgtype` query parameter in the URL. See
+URL section below for more on `msgtype`.
 
 ### Single Message
 
@@ -118,20 +141,21 @@ The above schema will be used as-is for the single message.
 
 ### Batch Message
 
-For a batch, the schema must be like this:
-```
+For the batch message, the schema must be like this:
+
+```protobuf
 message Batch {
   string type = 1;
   repeated Log batch = 2;
 }
 ```
 
-IMPORTANT:
-The `Batch` message type is part of `log.proto`, it's not a separate file!  
-You can choose any name for a batch message type.
+IMPORTANT: The `Batch` message type is part of `log.proto`, it is not a separate
+file! You can choose any name for a batch message type.
 
-The complete `log.proto` will be:
-```
+Here is the complete `log.proto` file:
+
+```protobuf
 syntax = "proto3";
 
 package service.logging;
@@ -165,50 +189,62 @@ message Batch {
 }
 ```
 
-For batch processing, the plugin looks for special members `type` and `batch`.  
+For batch processing, the plugin looks for special members `type` and `batch`.
 The `type` will indicate the message type of `batch` i.e. `Log` in this example.
 
-The type of `Batch` is `service.logging.Batch` and it will be the value of `msgtype` in the URL query.  
-The type of `batch` array is `service.logging.Log` and it will be the value of `type`.
+The type of `Batch` is `service.logging.Batch` and it will be the value of
+`msgtype` in the URL query. The type of `batch` array is `service.logging.Log`
+and it will be the value of `type`.
 
-The `google.protobuf.Any` type has not been used deliberately here.  
-It stores message type information with each message resulting in increase in size.  
-With the above approach, the type is stored only once for the whole batch.
+The `google.protobuf.Any` type has not been used here deliberately. It stores
+the message type with each message resulting in an increase in size. Refer to
+[protobuf-repeated-type-vs-any](https://github.com/iamAzeem/protobuf-repeated-type-vs-any)
+for a simple comparison. With the above approach, the type is stored only once
+for the whole batch.
 
 ### Endpoint (URL)
 
 For single message:
-```
+
+```text
 http://<ip>:<port>/<tag>?msgtype=<fully-qualified-message-type>
 ```
 
 For batch message:
-```
+
+```text
 http://<ip>:<port>/<tag>?msgtype=<fully-qualified-message-type-for-batch>&batch=true
 ```
 
-Without `batch=true` query parameter, the batch will be treated as a single message.
+Without `batch=true` query parameter, the batch will be treated as a single
+message.
 
-For example, for a log type `service.logging.Log` and its corresponding batch type `service.logging.Batch`:
+For example, for a log type `service.logging.Log` and its corresponding batch
+type `service.logging.Batch`, the URLs would be:
 
-Single:
-```
+For single message:
+
+```text
 http://localhost:8080/debug.test?msgtype=service.logging.Log
 ```
 
-Batch:
-```
+For batch message:
+
+```text
 http://localhost:8080/debug.test?msgtype=service.logging.Batch&batch=true
 ```
 
-**NOTE**: The values of query parameters (`msgtype`, `batch`) are case-sensitive!
+**NOTE**: The values of query parameters (`msgtype`, `batch`) are
+case-sensitive!
 
 ## Test Use-Case (`curl`)
 
-For a simple test use-case of events and their routing to [stdout](https://docs.fluentd.org/output/stdout) can be configured like this:
+For a simple use-case of incoming HTTP events and their routing to
+[stdout](https://docs.fluentd.org/output/stdout) may be configured like this:
 
 `fluent.conf`:
-```
+
+```text
 <source>
   @type       protobuf_http
   @id         protobuf_http_input
@@ -224,24 +260,36 @@ For a simple test use-case of events and their routing to [stdout](https://docs.
 
 <match debug.test>
   @type       stdout
-  @id         stdout_output
 </match>
 ```
 
 The incoming binary messages will be transformed to JSON for further consumption.
 
-#### Single Message
+### Single Message
+
+Test Parameters:
 
-Test Input Parameters:
-* data: `log.bin`, msgtype: `service.logging.Log`
+| input file | single message type   |
+|:----------:|:---------------------:|
+| `log.bin`  | `service.logging.Log` |
 
-Command:
-```
-$ curl -X POST -H "Content-Type: application/octet-stream" --data-binary "@/<path>/log.bin" "http://localhost:8080/debug.test?msgtype=service.logging.Log"
+URL:
+
+```bash
+http://localhost:8080/debug.test?msgtype=service.logging.Log
 ```
 
-`fluentd` Logs (Observe JSON at the end):
+`curl` command:
+
+```shell
+curl -X POST -H "Content-Type: application/octet-stream" \
+    --data-binary "@/<path>/log.bin" \
+    "http://localhost:8080/debug.test?msgtype=service.logging.Log"
 ```
+
+`fluentd` logs (Observe JSON at the end):
+
+```text
 2020-06-09 18:53:47 +0500 [info]: #0 [protobuf_http_input] [R] {binary} [127.0.0.1:41222, size: 86 bytes]
 2020-06-09 18:53:47 +0500 [warn]: #0 [protobuf_http_input] 'batch' not found in 'query_string' [msgtype=service.logging.Log]
 2020-06-09 18:53:47 +0500 [info]: #0 [protobuf_http_input] [S] {binary} [127.0.0.1:41222, msgtype: service.logging.Log, size: 86 bytes]
@@ -249,22 +297,38 @@ $ curl -X POST -H "Content-Type: application/octet-stream" --data-binary "@/<pat
 2020-06-09 18:53:47 +0500 [info]: #0 [protobuf_http_input] [S] {json} [127.0.0.1:41222, msgtype: service.logging.Log, size: 183 bytes]
 ```
 
-For Test Single Message Generation: https://github.com/iamAzeem/protobuf-log-sample
+For sample Single message generation, see
+[this](https://github.com/iamAzeem/protobuf-log-sample).
+
+### Batch Message
+
+Test Parameters:
 
-#### Batch Message
+| input file      | batch message type      | batch internal type   | messages |
+|:---------------:|:-----------------------:|:---------------------:|:--------:|
+| `logbatch2.bin` | `service.logging.Batch` | `service.logging.Log` | 2        |
+| `logbatch5.bin` | `service.logging.Batch` | `service.logging.Log` | 5        |
 
-Test Input Parameters:
-* data: `logbatch2.bin`, msgtype: `service.logging.Batch`, type: `service.logging.Log` [batch_size: 2 messages]
-* data: `logbatch5.bin`, msgtype: `service.logging.Batch`, type: `service.logging.Log` [batch_size: 5 messages]
+URL:
 
-Command (`logbatch2.bin`):
+```text
+http://localhost:8080/debug.test?msgtype=service.logging.Batch&batch=true
 ```
-$ curl -X POST -H "Content-Type: application/octet-stream" --data-binary "@/<path>/logbatch2.bin" "http://localhost:8080/debug.test?msgtype=service.logging.Batch&batch=true"
+
+**`logbatch2.bin`**
+
+`curl` command:
+
+```shell
+$ curl -X POST -H "Content-Type: application/octet-stream" \
+    --data-binary "@/<path>/logbatch2.bin" \
+    "http://localhost:8080/debug.test?msgtype=service.logging.Batch&batch=true"
 {"status":"Batch received! [batch_type: service.logging.Log, batch_size: 2 messages]"}
 ```
 
-`fluentd` Logs:
-```
+`fluentd` logs:
+
+```text
 2020-06-09 19:04:13 +0500 [info]: #0 [protobuf_http_input] [R] {binary} [127.0.0.1:41416, size: 207 bytes]
 2020-06-09 19:04:13 +0500 [info]: #0 [protobuf_http_input] [B] {binary} [127.0.0.1:41416, msgtype: service.logging.Batch, size: 207 bytes]
 2020-06-09 19:04:13 +0500 [info]: #0 [protobuf_http_input] [B] Emitting message stream/batch [batch_size: 2 messages]...
@@ -273,14 +337,20 @@ $ curl -X POST -H "Content-Type: application/octet-stream" --data-binary "@/<pat
 2020-06-09 19:04:13 +0500 [info]: #0 [protobuf_http_input] [B] {json} [127.0.0.1:41416, msgtype: service.logging.Batch] Batch received! [batch_type: service.logging.Log, batch_size: 2 messages]
 ```
 
-Command (`logbatch5.bin`):
-```
-$ curl -X POST -H "Content-Type: application/octet-stream" --data-binary "@/<path>/logbatch5.bin" "http://localhost:8080/debug.test?msgtype=service.logging.Batch&batch=true"
+**`logbatch5.bin`**
+
+`curl` command:
+
+```bash
+$ curl -X POST -H "Content-Type: application/octet-stream" \
+    --data-binary "@/<path>/logbatch5.bin" \
+    "http://localhost:8080/debug.test?msgtype=service.logging.Batch&batch=true"
 {"status":"Batch received! [batch_type: service.logging.Log, batch_size: 5 messages]"}
 ```
 
-`fluentd` Logs:
-```
+`fluentd` logs:
+
+```text
 2020-06-09 19:07:09 +0500 [info]: #0 [protobuf_http_input] [R] {binary} [127.0.0.1:41552, size: 486 bytes]
 2020-06-09 19:07:09 +0500 [info]: #0 [protobuf_http_input] [B] {binary} [127.0.0.1:41552, msgtype: service.logging.Batch, size: 486 bytes]
 2020-06-09 19:07:09 +0500 [info]: #0 [protobuf_http_input] [B] Emitting message stream/batch [batch_size: 5 messages]...
@@ -292,10 +362,19 @@ $ curl -X POST -H "Content-Type: application/octet-stream" --data-binary "@/<pat
 2020-06-09 19:07:09 +0500 [info]: #0 [protobuf_http_input] [B] {json} [127.0.0.1:41552, msgtype: service.logging.Batch] Batch received! [batch_type: service.logging.Log, batch_size: 5 messages]
 ```
 
-For Test Batch Message Generation: https://gist.github.com/iamAzeem/a8a24092132e1741a76956192f2104cc
+For sample Batch message generation, see
+[this](https://gist.github.com/iamAzeem/a8a24092132e1741a76956192f2104cc).
+
+## Contribute
+
+- Fork the project.
+- Check out the latest `main` branch.
+- Create a feature or bugfix branch from `main`.
+- Commit and push your changes.
+- Make sure to add and run tests locally: `bundle exec rake test`.
+- Run `rubocop` locally and fix all the lint warnings.
+- Submit the PR.
 
-## Copyright
+## License
 
-* Copyright&copy; 2020 [Azeem Sajid](https://www.linkedin.com/in/az33msajid/)
-* License
-  * Apache License, Version 2.0
+[Apache 2.0](LICENSE)

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen-string-literal: true
+
 require 'bundler'
 Bundler::GemHelper.install_tasks
 

data/fluent-plugin-protobuf-http.gemspec

@@ -1,9 +1,11 @@
+# frozen-string-literal: true
+
 lib = File.expand_path('../lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name           = 'fluent-plugin-protobuf-http'
-  spec.version        = '0.2.0'
+  spec.version        = '0.3.0'
   spec.authors        = ['Azeem Sajid']
   spec.email          = ['azeem.sajid@gmail.com']
 
@@ -20,8 +22,9 @@ Gem::Specification.new do |spec|
   spec.test_files     = test_files
   spec.require_paths  = ['lib']
 
-  spec.add_development_dependency 'bundler', '~> 1.14'
+  spec.add_development_dependency 'bundler', '~> 2.1', '>= 2.1.0'
   spec.add_development_dependency 'rake', '~> 12.0'
+  spec.add_development_dependency 'simplecov', '~> 0.12', '<= 0.12.2'
   spec.add_development_dependency 'test-unit', '~> 3.0'
   spec.add_runtime_dependency 'fluentd', ['>= 0.14.10', '< 2']
   spec.add_runtime_dependency 'google-protobuf', '~> 3.12', '>= 3.12.2'

data/lib/fluent/plugin/in_protobuf_http.rb

@@ -1,3 +1,5 @@
+# frozen-string-literal: true
+
 #
 # Copyright 2020 Azeem Sajid
 #
@@ -18,9 +20,11 @@ require 'fluent/config/error'
 require 'fluent/plugin_helper/http_server'
 require 'webrick/httputils'
 require 'json'
+require 'English'
 
 module Fluent
   module Plugin
+    # Implementation of HTTP input plugin for Protobuf
     class ProtobufHttpInput < Fluent::Plugin::Input
       Fluent::Plugin.register_input('protobuf_http', self)
 
@@ -46,10 +50,6 @@ module Fluent
         config_argument :protocol, :enum, list: %i[tcp tls], default: :tcp
       end
 
-      config_section :transform, required: false, multi: false, init: true, param_name: :transform_config do
-        config_argument :msgtype, :string
-      end
-
       def initialize
         super
 
@@ -61,16 +61,16 @@ module Fluent
       def compile_protos
         log.debug("Checking proto_dir [#{@proto_dir}]...")
 
-        path = File.expand_path(@proto_dir).freeze
+        path = File.expand_path(@proto_dir)
         raise Fluent::ConfigError, "protos_dir does not exist! [#{path}]" unless Dir.exist?(path)
 
-        @protos = Dir["#{path}/*.proto"].freeze
+        @protos = Dir["#{path}/*.proto"]
         raise Fluent::ConfigError, "Empty proto_dir! [#{path}]" unless @protos.any?
 
         log.info("Compiling .proto files [#{@protos.length}]...")
 
         `protoc --ruby_out=#{path} --proto_path=#{path} #{path}/*.proto`
-        raise Fluent::ConfigError, 'Could not compile! See error(s) above.' unless $?.success?
+        raise Fluent::ConfigError, 'Could not compile! See error(s) above.' unless $CHILD_STATUS.success?
 
         log.info("Compiled successfully:\n- #{@protos.join("\n- ")}")
 
@@ -82,8 +82,8 @@ module Fluent
       end
 
       def get_compiled_proto(proto)
-        proto_suffix = '.proto'.freeze
-        compiled_proto_suffix = '_pb.rb'.freeze
+        proto_suffix = '.proto'
+        compiled_proto_suffix = '_pb.rb'
 
         compiled_proto = proto.chomp(proto_suffix) + compiled_proto_suffix
         raise Fluent::ConfigError, "Compiled proto not found! [#{compiled_proto}]" unless File.file?(compiled_proto)
@@ -117,7 +117,7 @@ module Fluent
         msg_types = []
         File.foreach(compiled_proto) do |line|
           if line.lstrip.start_with?('add_message')
-            msg_type = line[/"([^"]*)"/, 1].freeze # regex: <add_message> 'msg_type' <do>
+            msg_type = line[/"([^"]*)"/, 1] # regex: <add_message> 'msg_type' <do>
             msg_types.push(msg_type) unless msg_type.nil?
           end
         end
@@ -152,15 +152,13 @@ module Fluent
           tls_opts = @transport_config.to_h
         end
 
-        log.warn("#{@transform_config.to_h}")
-
         log.info("Starting protobuf #{proto == :tcp ? 'HTTP' : 'HTTPS'} server [#{@bind}:#{@port}]...")
         log.debug("TLS configuration:\n#{tls_opts}") if tls_opts
 
         http_server_create_http_server(:protobuf_server, addr: @bind, port: @port, logger: log, proto: proto, tls_opts: tls_opts) do |server|
           server.post("/#{tag}") do |req|
-            peeraddr = "#{req.peeraddr[2]}:#{req.peeraddr[1]}".freeze # ip:port
-            serialized_msg = req.body.freeze
+            peeraddr = "#{req.peeraddr[2]}:#{req.peeraddr[1]}" # ip:port
+            serialized_msg = req.body
 
             log.info("[R] {#{@in_mode}} [#{peeraddr}, size: #{serialized_msg.length} bytes]")
             log.debug("Dumping serialized message [#{serialized_msg.length} bytes]:\n#{serialized_msg}")
@@ -168,7 +166,7 @@ module Fluent
             content_type = req.header['content-type'][0]
 
             unless valid_content_type?(content_type)
-              status = "Invalid 'Content-Type' header! [#{content_type}]".freeze
+              status = "Invalid 'Content-Type' header! [#{content_type}]"
               log.warn("[X] Message rejected! [#{peeraddr}] #{status}")
               next [400, { 'Content-Type' => 'application/json', 'Connection' => 'close' }, { 'status' => status }.to_json]
             end
@@ -177,7 +175,7 @@ module Fluent
 
             msgtype, batch = get_query_params(req.query_string)
             unless @msgclass_lookup.key?(msgtype)
-              status = "Invalid 'msgtype' in 'query_string'! [#{msgtype}]".freeze
+              status = "Invalid 'msgtype' in 'query_string'! [#{msgtype}]"
               log.warn("[X] Message rejected! [#{peeraddr}] #{status}")
               next [400, { 'Content-Type' => 'application/json', 'Connection' => 'close' }, { 'status' => status }.to_json]
             end
@@ -187,7 +185,7 @@ module Fluent
             deserialized_msg = deserialize_msg(msgtype, serialized_msg)
 
             if deserialized_msg.nil?
-              status = "Incompatible message! [msgtype: #{msgtype}, size: #{serialized_msg.length} bytes]".freeze
+              status = "Incompatible message! [msgtype: #{msgtype}, size: #{serialized_msg.length} bytes]"
               log.warn("[X] Message rejected! [#{peeraddr}] #{status}")
               next [400, { 'Content-Type' => 'application/json', 'Connection' => 'close' }, { 'status' => status }.to_json]
             end
@@ -214,7 +212,7 @@ module Fluent
             log.info("[B] {#{@in_mode}} [#{peeraddr}, msgtype: #{msgtype}, size: #{serialized_msg.length} bytes]")
 
             if deserialized_msg.type.nil? || deserialized_msg.batch.nil? || deserialized_msg.batch.empty?
-              status = "Invalid 'batch' message! [msgtype: #{msgtype}, size: #{serialized_msg.length} bytes]".freeze
+              status = "Invalid 'batch' message! [msgtype: #{msgtype}, size: #{serialized_msg.length} bytes]"
               log.warn("[X] Message rejected! [#{peeraddr}] #{status}")
               next [400, { 'Content-Type' => 'application/json', 'Connection' => 'close' }, { 'status' => status }.to_json]
             end
@@ -234,7 +232,7 @@ module Fluent
 
             router.emit_stream(@tag, stream)
 
-            status = "Batch received! [batch_type: #{batch_type}, batch_size: #{batch_size} messages]".freeze
+            status = "Batch received! [batch_type: #{batch_type}, batch_size: #{batch_size} messages]"
             log.info("[B] {#{@out_mode}} [#{peeraddr}, msgtype: #{msgtype}] #{status}")
             [200, { 'Content-Type' => 'application/json', 'Connection' => 'close' }, { 'status' => status }.to_json]
           end
@@ -242,8 +240,8 @@ module Fluent
       end
 
       def valid_content_type?(content_type)
-        hdr_binary = 'application/octet-stream'.freeze
-        hdr_json = 'application/json'.freeze
+        hdr_binary = 'application/octet-stream'
+        hdr_json = 'application/json'
 
         case @in_mode
         when :binary
@@ -284,7 +282,7 @@ module Fluent
         rescue Google::Protobuf::ParseError => e
           log.error("Incompatible message! [msgtype: #{msgtype}, size: #{serialized_msg.length} bytes] #{e}")
           nil
-        rescue => e
+        rescue StandardError => e
           log.error("Deserializaton failed! Error: #{e}")
           nil
         end
@@ -300,7 +298,7 @@ module Fluent
           when :json
             msgclass.encode_json(deserialized_msg)
           end
-        rescue => e
+        rescue StandardError => e
           log.error("Serialization failed! [msgtype: #{msgtype}, msg: #{deserialized_msg}] Error: #{e}")
           nil
         end

data/test/data/log.bin

@@ -0,0 +1,3 @@
+
+%
+����192.168.xxx.xxxtest"test
+This is a test log generated by [./log.rb].

data/test/data/log.json

@@ -0,0 +1 @@
+{"context":{"timestamp":"2020-06-01T16:24:19Z","hostOrIp":"192.168.xxx.xxx","serviceName":"test","user":"test"},"level":"INFO","message":"This is a test log generated by [./log.rb]."}

data/test/data/logbatch5.bin

@@ -0,0 +1,12 @@
+
+service.logging.Log[
+%
+����192.168.xxx.xxxtest"test
0This is a test log generated by [./logbatch.rb].[
+%
+����192.168.xxx.xxxtest"test
0This is a test log generated by [./logbatch.rb].[
+%
+����192.168.xxx.xxxtest"test
0This is a test log generated by [./logbatch.rb].[
+%
+����192.168.xxx.xxxtest"test
0This is a test log generated by [./logbatch.rb].[
+%
+����192.168.xxx.xxxtest"test
0This is a test log generated by [./logbatch.rb].

data/test/data/protos/log.proto

@@ -0,0 +1,31 @@
+syntax = "proto3";
+
+package service.logging;
+
+import "google/protobuf/timestamp.proto";
+
+message Log {
+  message Context {
+    google.protobuf.Timestamp timestamp = 1;
+    string host_or_ip = 2;
+    string service_name = 3;
+    string user = 4;
+  }
+
+  enum Level {
+    DEBUG = 0;
+    INFO = 1;
+    WARN = 2;
+    ERROR = 3;
+    FATAL = 4;
+  }
+
+  Context context = 1;
+  Level level = 2;
+  string message = 3;
+}
+
+message Batch {
+  string type = 1;
+  repeated Log batch = 2;
+}

data/test/helper.rb

@@ -1,8 +1,13 @@
-$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
-require "test-unit"
-require "fluent/test"
-require "fluent/test/driver/input"
-require "fluent/test/helpers"
+# frozen-string-literal: true
+
+require 'simplecov'
+SimpleCov.start
+
+$LOAD_PATH.unshift(File.expand_path('..', __dir__))
+require 'test-unit'
+require 'fluent/test'
+require 'fluent/test/driver/input'
+require 'fluent/test/helpers'
 
 Test::Unit::TestCase.include(Fluent::Test::Helpers)
 Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_in_protobuf_http.rb

@@ -1,18 +1,152 @@
-require "helper"
-require "fluent/plugin/in_protobuf_http.rb"
+# frozen-string-literal: true
 
+require 'helper'
+require 'fluent/plugin/in_protobuf_http'
+require 'net/http'
+
+# Implementation of Test Class
 class ProtobufHttpInputTest < Test::Unit::TestCase
   setup do
     Fluent::Test.setup
+    @log_bin = File.open('./test/data/log.bin', 'rb') { |f| f.read }
+    @log_json = File.open('./test/data/log.json', 'r') { |f| f.read }
+    @log_bin_batch = File.open('./test/data/logbatch5.bin', 'rb') { |f| f.read }
+  end
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::ProtobufHttpInput).configure(conf)
   end
 
-  test "failure" do
-    flunk
+  sub_test_case 'configure' do
+    test 'test default configuration' do
+      conf = %(
+        proto_dir ./test/data/protos
+        tag       test
+      )
+      driver = create_driver(conf)
+      plugin = driver.instance
+      assert_equal plugin.class, Fluent::Plugin::ProtobufHttpInput
+      assert_equal plugin.bind, '0.0.0.0'
+      assert_equal plugin.port, 8080
+      assert_equal plugin.in_mode, :binary
+      assert_equal plugin.out_mode, :binary
+    end
+  end
+
+  sub_test_case 'route#emit' do
+    conf = %(
+      proto_dir   ./test/data/protos
+      tag         test
+      in_mode     binary
+      out_mode    json
+    )
+
+    test 'test invalid msgtype in query string i.e. empty or mismatch' do
+      driver = create_driver(conf)
+      res_codes = []
+      driver.run do
+        path = '/test'
+        res = post(path, @log_bin)
+        res_codes << res.code
+      end
+      assert_equal 1, res_codes.size
+      assert_equal '400', res_codes[0]
+    end
+
+    test 'test incoming type mismatch [in_mode != Content-Type]' do
+      conf = %(
+        proto_dir   ./test/data/protos
+        tag         test
+        in_mode     binary
+        out_mode    json
+      )
+      driver = create_driver(conf)
+      res_codes = []
+      driver.run do
+        path = '/test?msgtype=service.logging.Log'
+        res = post(path, @log_bin, :json)
+        res_codes << res.code
+      end
+      assert_equal 1, res_codes.size
+      assert_equal '400', res_codes[0]
+    end
+
+    test 'test single message (Binary to JSON)' do
+      driver = create_driver(conf)
+      res_codes = []
+      driver.run do
+        path = '/test?msgtype=service.logging.Log'
+        res = post(path, @log_bin)
+        res_codes << res.code
+      end
+      assert_equal 1, res_codes.size
+      assert_equal '200', res_codes[0]
+    end
+
+    test 'test single message (JSON to Binary)' do
+      conf = %(
+        proto_dir   ./test/data/protos
+        tag         test
+        in_mode     json
+        out_mode    binary
+      )
+      driver = create_driver(conf)
+      res_codes = []
+      driver.run do
+        path = '/test?msgtype=service.logging.Log'
+        res = post(path, @log_json, :json)
+        res_codes << res.code
+      end
+      assert_equal 1, res_codes.size
+      assert_equal '200', res_codes[0]
+    end
+
+    test 'test batch messages (Binary to JSON)' do
+      conf = %(
+        proto_dir   ./test/data/protos
+        tag         test
+        in_mode     binary
+        out_mode    json
+      )
+      driver = create_driver(conf)
+      res_codes = []
+      driver.run do
+        path = '/test?msgtype=service.logging.Batch&batch=true'
+        res = post(path, @log_bin_batch)
+        res_codes << res.code
+      end
+      assert_equal 1, res_codes.size
+      assert_equal '200', res_codes[0]
+    end
+
+    test 'test incompatible message' do
+      conf = %(
+        proto_dir   ./test/data/protos
+        tag         test
+        in_mode     binary
+        out_mode    json
+      )
+      driver = create_driver(conf)
+      res_codes = []
+      driver.run do
+        path = '/test?msgtype=service.logging.Log'
+        res = post(path, @log_bin_batch)
+        res_codes << res.code
+      end
+      assert_equal 1, res_codes.size
+      assert_equal '400', res_codes[0]
+    end
   end
 
   private
 
-  def create_driver(conf)
-    Fluent::Test::Driver::Input.new(Fluent::Plugin::ProtobufHttpInput).configure(conf)
+  def post(path, body, type = :binary)
+    http = Net::HTTP.new('127.0.0.1', 8080)
+    content_type = 'application/octet-stream'
+    content_type = 'application/json' if type == :json
+    header = { 'Content-Type' => content_type }
+    req = Net::HTTP::Post.new(path, header)
+    req.body = body
+    http.request(req)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-protobuf-http
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Azeem Sajid
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-11 00:00:00.000000000 Z
+date: 2021-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -38,6 +44,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: 0.12.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: 0.12.2
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -100,13 +126,19 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
+- ".rubocop.yml"
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
 - fluent-plugin-protobuf-http.gemspec
 - lib/fluent/plugin/in_protobuf_http.rb
+- test/data/log.bin
+- test/data/log.json
+- test/data/logbatch5.bin
+- test/data/protos/log.proto
 - test/helper.rb
 - test/plugin/test_in_protobuf_http.rb
 homepage: https://github.com/iamAzeem/fluent-plugin-protobuf-http
@@ -134,5 +166,9 @@ signing_key:
 specification_version: 4
 summary: fluentd HTTP Input Plugin for Protocol Buffers
 test_files:
+- test/data/log.bin
+- test/data/log.json
+- test/data/logbatch5.bin
+- test/data/protos/log.proto
 - test/helper.rb
 - test/plugin/test_in_protobuf_http.rb