fluent-plugin-unix-client 0.1.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cc18f5099028958215b671685ea41ebcd5c6754c0b1ee0eb46b3fc87f0b08226
-  data.tar.gz: a0d5bbe77d1ccd49d9c5c7a452a3702397fdfd9c41da6f81fd7a35e509b43ba5
+  metadata.gz: e730565e1f497509cb2d5e579527d5b500ba3240312fa0bb6527972db7e57ff4
+  data.tar.gz: 9d4a45f7a0c3cbf6b30b00be745dc624ed79d5b7e10f9753c7af2a5728148cca
 SHA512:
-  metadata.gz: f28da19e7f6291fad09fd1a19373d241db41c9d29eedc53b79e0f6432b7d9ad499dea0d5ba5d65440a2f1030e88888f63cbe3086f94eff21334c40811c2fb5c2
-  data.tar.gz: 22b091803d9db3ae70d1df0fcf3d9a2e7077bbf83b62274b54512a3c9b9900f304e75308396e8f3742536f15dfac540a489da2f1c575c74d7734c389d9c00658
+  metadata.gz: 3a186a0de2333d95f32162f54b38fcab619b2e113122d0e08c7d10338aa9e3b64eb6c838ef34828772fec956d4c255bfd11be5c84af5aa703a631381bb7800b5
+  data.tar.gz: dfd15243ad040f815bd5c9c5dccb24c72bc57cadcbacc118cad2f4a1f0a597bfcbcc4d801e3294d63af04fbd440555d972165a33a08a88ef07a20a19e0c894a9

data/README.md

@@ -48,8 +48,20 @@ The payload is read up to this character.
 
 Default value: `"\n"` (newline).
 
+### format_json (bool) (optional)
+
+When recieved JSON data splitted by the delimiter is not completed, like '[{...},', trim '[', ']' and ',' characters to format.
+
+Please see `Sample` below for details.
+
+Default value: false.
+
 ## Sample
 
+### For JSON
+
+Config:
+
 ```
 <source>
   @type unix_client
@@ -66,6 +78,53 @@ Default value: `"\n"` (newline).
 </match>
 ```
 
+Assumed Data:
+
+* ndjson
+```
+{"key":0}\n
+{"key":0}\n
+...
+```
+
+* JSON list
+```
+[{"key":0}, {"key":0}, ...]\n
+[{"key":0}, {"key":0}, ...]\n
+...
+```
+
+### Use `format_json`
+
+Config:
+
+```
+<source>
+  @type unix_client
+  tag debug.unix_client
+  path /tmp/unix.sock
+  <parse>
+    @type json
+  </parse>
+  delimiter "\n"
+  format_json true
+</source>
+
+<match debug.**>
+  @type stdout
+</match>
+```
+
+Assumed Data:
+
+```
+[{"key":0},\n
+{"key":0},\n
+...
+{"key":0}]\n
+...
+```
+
 ## Specification
 
 * This recieves data from UNIX domain socket which **is opened by another application**.

data/fluent-plugin-unix-client.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-unix-client"
-  spec.version = "0.1.0"
+  spec.version = "1.0.0"
   spec.authors = ["daipom"]
   spec.email   = ["reangoapyththeorem@gmail.com"]
 

data/lib/fluent/plugin/in_unix_client.rb

@@ -15,6 +15,7 @@
 
 require "fluent/plugin/input"
 require 'socket'
+require "json"
 
 module Fluent
   module Plugin
@@ -29,11 +30,19 @@ module Fluent
       config_param :path, :string
       desc 'The payload is read up to this character.'
       config_param :delimiter, :string, default: "\n"
+      desc "When recieved JSON data splitted by the delimiter is not completed, like '[{...},'," \
+           " trim '[', ']' and ',' characters to format."
+      config_param :format_json, :bool, default: false
 
       def configure(conf)
         super
         @parser = parser_create
-        @socket_handler = SocketHandler.new(@path, delimiter: @delimiter, log: log)
+        @socket_handler = SocketHandler.new(
+          @path,
+          delimiter: @delimiter,
+          format_json: @format_json,
+          log: log,
+        )
       end
 
       def start
@@ -60,21 +69,34 @@ module Fluent
 
         raw_records.each do |raw_record|
           @parser.parse(raw_record) do |time, record|
-            router.emit(@tag, time, record)
+            emit_one_parsed(time, record)
           end
         end
       end
+
+      def emit_one_parsed(time, record)
+        case record
+        when Array
+          es = Fluent::MultiEventStream.new
+          record.each do |e|
+            es.add(time, e)
+          end
+          router.emit_stream(@tag, es)
+        else
+          router.emit(@tag, time, record)
+        end
+      end
     end
 
 
     class SocketHandler
       MAX_LENGTH_RECEIVE_ONCE = 10000
 
-      def initialize(path, delimiter: "\n", log: nil)
+      def initialize(path, delimiter: "\n", format_json: false, log: nil)
         @path = path
         @log = log
         @socket = nil
-        @buf = Buffer.new(delimiter)
+        @buf = Buffer.new(delimiter, format_json: format_json)
       end
 
       def connected?
@@ -142,9 +164,10 @@ module Fluent
 
 
     class Buffer
-      def initialize(delimiter)
+      def initialize(delimiter, format_json: false)
         @buf = ""
         @delimiter = delimiter
+        @format_json = format_json
       end
 
       def add(data)
@@ -160,7 +183,8 @@ module Fluent
 
         pos_read = 0
         while pos_next_delimiter = @buf.index(@delimiter, pos_read)
-          records << @buf[pos_read...pos_next_delimiter]
+          fixed = fix_format(@buf[pos_read...pos_next_delimiter])
+          records << fixed unless fixed.empty?
           pos_read = pos_next_delimiter + @delimiter.size
         end
 
@@ -168,6 +192,41 @@ module Fluent
 
         records
       end
+
+      private
+
+      def fix_format(record)
+        return record if record.empty?
+        return record unless @format_json
+
+        fix_uncompleted_json(record)
+      end
+
+      def fix_uncompleted_json(record)
+        return record if is_correct_json(record)
+
+        # Assume uncompleted JSON such as "[{...},", "{...},", or "{...}]"
+
+        if record[0] == "["
+          record.slice!(0)
+          return record if record.empty?
+        end
+
+        if record[-1] == "," || record[-1] == "]"
+          record.slice!(-1)
+          return record if record.empty?
+        end
+
+        record
+      end
+
+      def is_correct_json(record)
+        # Just to check the format
+        JSON.parse(record)
+        return true
+      rescue JSON::ParserError
+        return false
+      end
     end
   end
 end

data/test/plugin/test_in_unix_client.rb

@@ -9,12 +9,11 @@ require_relative "./unix_server.rb"
 class UnixClientInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    @thread = nil
+    @server_thread = nil
   end
 
   def teardown
-    @thread.kill if @thread
-    @thread = nil
+    stop_server
     FileUtils.rm_rf(TMP_DIR)
   end
 
@@ -32,6 +31,14 @@ class UnixClientInputTest < Test::Unit::TestCase
     d = create_driver(config_with_json_parser)
     assert_equal "unix_client", d.instance.tag
     assert_equal "#{TMP_DIR}/socket.sock", d.instance.path
+    assert_equal false, d.instance.format_json
+  end
+
+  def test_configure_with_format_json
+    d = create_driver(config_with_json_parser_and_format_json)
+    assert_equal "unix_client", d.instance.tag
+    assert_equal "#{TMP_DIR}/socket.sock", d.instance.path
+    assert_equal true, d.instance.format_json
   end
 
   def test_receive_json
@@ -100,6 +107,67 @@ class UnixClientInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_receive_json_list
+    d = create_driver(config_with_json_parser_and_format_json)
+    path = d.instance.path
+    delimiter = "\n"
+
+    msgs = ["hoge", "fuga", "foo"]
+
+    start_server(path)
+
+    d.run(expect_records: 3, timeout: 10) do
+      sleep 1
+      UNIXSocket.open(path) do |sock|
+        sock.write("[")
+        sock.write(JSON.generate(raw_data(msg: msgs[0])))
+        sock.write(",")
+        sock.write(delimiter)
+
+        sock.write(JSON.generate(raw_data(msg: msgs[1])))
+        sock.write(",")
+        sock.write(delimiter)
+
+        sock.write(JSON.generate(raw_data(msg: msgs[2])))
+        sock.write(delimiter)
+        sock.write("]")
+        sock.write(delimiter)
+      end
+    end
+
+    assert_equal 3, d.events.length
+
+    d.events.each_with_index do |event, i|
+      assert_equal msgs[i], event[2]["msg"]
+    end
+  end
+
+  def test_receive_json_list_with_one_delimiter
+    d = create_driver(config_with_json_parser_and_format_json)
+    path = d.instance.path
+    delimiter = "\n"
+
+    msgs = ["hoge", "fuga", "foo"]
+
+    start_server(path)
+
+    d.run(expect_records: 3, timeout: 10) do
+      sleep 1
+      data = msgs.map {|msg| raw_data(msg: msg)}
+
+      UNIXSocket.open(path) do |sock|
+        sock.write(JSON.generate(data))
+        sock.write(delimiter)
+      end
+    end
+
+    assert_equal 3, d.events.length
+
+    d.events.each_with_index do |event, i|
+      assert_equal msgs[i], event[2]["msg"]
+    end
+  end
+
   private
 
   def create_driver(conf)
@@ -124,24 +192,43 @@ class UnixClientInputTest < Test::Unit::TestCase
     !
   end
 
+  def config_with_json_parser_and_format_json
+    BASE_CONFIG + %!
+      format_json true
+      <parse>
+        @type json
+      </parse>
+    !
+  end
+
   def start_server(path)
-    @thread = Thread.new do
+    @server_thread = Thread.new do
       server = UnixBroadcastServer.new(path)
       server.run
     end
     sleep 1
   end
 
+  def stop_server
+    if @server_thread
+      @server_thread.kill if @server_thread
+      @server_thread.join
+      @server_thread = nil
+    end
+  end
+
   def send_json(path, time: nil, msg: DEFAULT_MSG, delimiter: "\n")
-    msg = JSON.generate(
-      {
-        "time" => time.nil? ? Time.now.to_i : time,
-        "msg" => msg
-      }
-    )
+    msg = JSON.generate(raw_data(time: time, msg: msg))
     UNIXSocket.open(path) do |sock|
       sock.write(msg)
       sock.write(delimiter)
     end
   end
+
+  def raw_data(time: nil, msg: DEFAULT_MSG)
+    {
+      "time" => time.nil? ? Time.now.to_i : time,
+      "msg" => msg
+    }
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-unix-client
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.0.0
 platform: ruby
 authors:
 - daipom
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-31 00:00:00.000000000 Z
+date: 2022-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -72,7 +72,7 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
-description:
+description: 
 email:
 - reangoapyththeorem@gmail.com
 executables: []
@@ -93,7 +93,7 @@ homepage: https://github.com/daipom/fluent-plugin-unix-client
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -108,8 +108,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.15
-signing_key:
+rubygems_version: 3.1.6
+signing_key: 
 specification_version: 4
 summary: Fluentd Input plugin to receive data from UNIX domain socket. This is a client
   version of the default `unix` input plugin.