fluent-plugin-directory 0.1.0 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e169ba627338c24ee1b4b2fa822d5a089847133e28515e975d90beaf2c72c9ba
-  data.tar.gz: '09c89e36f8f8577abf33ab004436233ff97a468d14ac8a5e8aa6eee8e307d35e'
+  metadata.gz: 55d77adfb2d0512e04037176adff4a2b82e16cf9264efce510eb61d88af5c64c
+  data.tar.gz: cc44c79f31f7194f6c45080cff9a4b6ed8cf6f3ba55f7ef827ea1a88d0d94ef0
 SHA512:
-  metadata.gz: fb481b952c173b9d779931ef69dedb2aae46dcfe9bf1030774836b8c8cf95bdf64cd3295a650e151028b67b0b19f76bd48189d3064c9f42cd3d2811dde083683
-  data.tar.gz: 5192a3f3c29a018662c007e2c9fe320d71837a84c312ce725250c38207eeae79f962787cf425e1a2dcc81931f9bf05c1827dd1f58b4745b14f69421e402b5b5b
+  metadata.gz: ff2e8b35d7118616430406a4a6740ce9636d05c64fd2ec48e3a40f4108d2c927b2a6394a4ad1f7f58bfa579ab2dff5b5970cabeac6b7309ef001154d467c5233
+  data.tar.gz: 9e83689cd6e2da654949c702500798ded723e2d4f2db5c0a4635c2b165f483301faf62c0009761d2046862636e972a9b13cf98421acce63b0a0b0adf4a54aeb3

data/README.md

@@ -26,16 +26,34 @@ $ bundle
 
 ## Configuration
 
-You can generate configuration template:
+### content_key (string) (optional)
 
-```
-$ fluent-plugin-config-format input directory
-```
+The field where the content of the file is stored in the output event.
+
+Default value: `content`.
+
+### filename_key (string) (optional)
+
+The field where the name of the file is stored in the output event.
+
+Default value: `filename`.
+
+### path (string) (required)
+
+The path of the folder to scan.
+
+### run_interval (integer) (optional)
+
+The interval (in seconds) to wait between scans.
+
+Default value: `60`.
+
+### tag (string) (required)
 
-You can copy and paste generated documents here.
+The tag added to the output event.
 
 ## Copyright
 
-- Copyright(c) 2021- TODO: Write your name
+- Copyright(c) 2021- Rémy DUTHU
 - License
   - Apache License, Version 2.0

data/fluent-plugin-directory.gemspec

@@ -3,8 +3,8 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-directory"
-  spec.version = "0.1.0"
-  spec.authors = ["remyduthu@icloud.com"]
+  spec.version = "0.1.4"
+  spec.authors = ["Rémy DUTHU"]
   spec.email   = ["remyduthu@icloud.com"]
 
   spec.summary       = "A Fluentd input plugin to scan files recurrently from a directory"

data/lib/fluent/plugin/in_directory.rb

@@ -1,5 +1,5 @@
 #
-# Copyright 2021- TODO: Write your name
+# Copyright 2021- Rémy DUTHU
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,19 +20,21 @@ module Fluent
     class DirectoryInput < Fluent::Plugin::Input
       Fluent::Plugin.register_input('directory', self)
 
-      desc 'The name of the key whose value is the content of the file.'
-      config_param :content_key, :string, default: 'content'
+      helpers :timer
+
+      desc 'The maximum number of files processed in each run.'
+      config_param :batch_size, :integer, default: 10
 
-      desc 'The extension that will be added to the processed files.'
-      config_param :extension, :string, default: '.done'
+      desc 'The field where the content of the file is stored in the output event.'
+      config_param :content_key, :string, default: 'content'
 
-      desc 'The name of the key whose value is the name of the file.'
+      desc 'The field where the name of the file is stored in the output event.'
       config_param :filename_key, :string, default: 'filename'
 
-      desc 'The path of the folder to be scanned by the plugin.'
+      desc 'The path of the folder to scan.'
       config_param :path, :string
 
-      desc 'The time interval (in seconds) to wait between scans.'
+      desc 'The interval (in seconds) to wait between scans.'
       config_param :run_interval, :integer, default: 60
 
       desc 'The tag added to the output event.'
@@ -41,9 +43,9 @@ module Fluent
       def start
         super
 
-        begin
-          # Scan files indefinitely
-          loop do
+        # See: https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-timer
+        timer_execute(:directory_timer, @run_interval) do
+          begin
             # Use a stream to submit multiple events at the same time
             multiEventStream = MultiEventStream.new
 
@@ -51,29 +53,28 @@ module Fluent
             time = Fluent::Engine.now
 
             # Read filenames in the directory
-            Dir.glob(@path + '/*') do |filename|
-              # Ignore already processed files
-              next if filename.end_with? @extension
-
-              # Add the record to the stream
-              multiEventStream.add(
-                time,
-                { @content_key => File.read(filename), @file_key => filename }
-              )
-
-              # Mark the file as processed
-              File.rename(filename, filename + @extension)
-            end
+            Dir
+              .glob(@path + '/*')
+              .take(@batch_size)
+              .each do |filename|
+                # Add the record to the stream
+                multiEventStream.add(
+                  time,
+                  {
+                    @content_key => File.read(filename),
+                    @filename_key => filename
+                  }
+                )
+
+                # Remove the file
+                File.delete(filename)
+              end
 
             # Send the events
             router.emit_stream(tag, multiEventStream)
-
-            # Wait before the next scan
-            sleep(@run_interval)
+          rescue StandardError
+            yield(nil, nil)
           end
-        rescue Exception => e
-          $log.warn 'Directory input error: ', e
-          $log.debug_backtrace(e.backtrace)
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-directory
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.4
 platform: ruby
 authors:
-- remyduthu@icloud.com
+- Rémy DUTHU
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-15 00:00:00.000000000 Z
+date: 2021-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -106,7 +106,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: A Fluentd input plugin to scan files recurrently from a directory