fluent-plugin-input-static-file 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f01a3ed3a9303da14101143f5a9c84299956df9c
-  data.tar.gz: 4d03cf0691b33ecc441c1f5dacba8c0ea2af644b
+  metadata.gz: e582c208330d7349f6b7b3803feb0210fbb38674
+  data.tar.gz: 54369be36d824d7e225d266e8f1b43e2f0ad5fb9
 SHA512:
-  metadata.gz: 6592ffb28e7008d905704af1af9b3855bd3ab5ff82cf6dae3aed3e7980789346a39e7cd73b013219347ce74e8218117feedefef4f1823a5a93415d967bd2ba50
-  data.tar.gz: 59244ecbfbb374ee3a464a8e031521cb3121afeaad85ef0564413c2fa96227a8533cf9a85029728f7b13ba54c64837f712e0243e412cb4b3c63d81310db96a6c
+  metadata.gz: fd5230aee29627eb7f1f97c90856c4624d2b70e8e9b219c8f262d3cea6bcb9303b6bd5e542d6b632585d17204f2035904ee514e847279529e6b5c7eafc049bbc
+  data.tar.gz: 663c4fae667de7cc00d61959ff15b0ce9867bdece38dd321d247a4f353bd2a1028917b70c7d9a53e3cb4cfc505ce661b7700ef4e014a4025389b905f74d4b48f

data/.ruby-version

@@ -0,0 +1 @@
+2.4.10

data/lib/fluent/plugin/in_static_file/file_tracker.rb

@@ -77,7 +77,7 @@ module Fluent
 
             @file.pos = 0
             @file.truncate(0)
-            @file.write(@cache.values.maps(&:to_file_tracker_entry_format).join)
+            @file.write(@cache.values.map(&:to_file_tracker_entry_format).join)
           end
         end
       end

data/lib/fluent/plugin/in_static_file.rb

@@ -15,14 +15,14 @@ module Fluent
   module Plugin
     # InStaticFile is an input plugin for file with static content
     class InStaticFile < Fluent::Plugin::Input
-      Fluent::Plugin.register_input("static_file", self)
+      PLUGIN_NAME = "static_file"
+
+      Fluent::Plugin.register_input(PLUGIN_NAME, self)
 
       helpers :timer, :compat_parameters, :parser
 
       RESERVED_CHARS = ["/", "*", "%"].freeze
 
-      # FIXME: path_key
-
       desc "The paths to read. Multiple paths can be specified, separated by comma."
       config_param :path, :string
       desc "path delimiter used for spliting path config"
@@ -40,12 +40,17 @@ module Fluent
 
       desc "The tag of the event."
       config_param :tag, :string
+      desc "Add the log path being tailed to records. Specify the field name to be used."
+      config_param :path_key, :string, default: nil
 
       desc "Fluentd will record the position it last read into this file."
       config_param :pos_file, :string, default: nil
       desc "Follow inodes instead of following file names. Guarantees more stable delivery and allows to use * in path pattern with rotating files"
       config_param :follow_inodes, :bool, default: false
 
+      desc "When processed move to another location"
+      config_param :archive_to, :string, default: nil
+
       def initialize
         super
 
@@ -70,19 +75,22 @@ module Fluent
         end
 
         @paths = @path.split(@path_delimiter).map(&:strip).uniq
-        raise Fluent::ConfigError, "static_file: 'path' parameter is required on static_file input" if @paths.empty?
+        if @paths.empty?
+          raise Fluent::ConfigError,
+                "#{PLUGIN_NAME}: 'path' parameter is required on #{PLUGIN_NAME} input"
+        end
 
         if @pos_file
           if @variable_store.key?(@pos_file) && !called_in_test?
             plugin_id_using_this_path = @variable_store[@pos_file]
             raise Fluent::ConfigError,
-                  "Other 'in_static_file' plugin already use same pos_file path: plugin_id = #{plugin_id_using_this_path}, pos_file path = #{@pos_file}"
+                  "Other '#{PLUGIN_NAME}' plugin already use same pos_file path: plugin_id = #{plugin_id_using_this_path}, pos_file path = #{@pos_file}"
           end
           @variable_store[@pos_file] = plugin_id
         else
           raise Fluent::ConfigError, "Can't follow inodes without pos_file configuration parameter" if @follow_inodes
 
-          log.warn "'pos_file PATH' parameter is not set to a 'static_file' source."
+          log.warn "'pos_file PATH' parameter is not set to a '#{PLUGIN_NAME}' source."
           log.warn "this parameter is highly recommended to save the file status."
         end
 
@@ -188,31 +196,53 @@ module Fluent
       end
 
       def untrack_files(files_info)
-        log.debug("static_file: untrack files: #{files_info.keys}")
+        log.debug("#{PLUGIN_NAME}: untrack files: #{files_info.keys}")
         files_info.each do |_id, file_info|
           @file_tracker.remove(file_info)
         end
       end
 
       def process_files(files_info)
-        log.debug("static_file: process files: #{files_info.keys}")
+        log.debug("#{PLUGIN_NAME}: process files: #{files_info.keys}")
         files_info.each do |_id, file_info|
           process_file(file_info)
         end
       end
 
       def process_file(file_info)
-        log.debug("static_file: process file: #{file_info.path}")
+        log.debug("#{PLUGIN_NAME}: process file: #{file_info.path}")
 
         return if @file_tracker.has?(file_info)
 
         File.open(file_info.path, "rb") do |f|
           @parser.parse(f) do |time, record|
+            record[@path_key] ||= file_info.path unless @path_key.nil?
             router.emit(tag, time, record)
           end
         end
 
         @file_tracker.add(file_info)
+
+        archive(file_info)
+      end
+
+      def archive(file_info)
+        return if @archive_to.nil?
+
+        file_name = File.basename(file_info.path)
+        target_path = format(@archive_to, file_name)
+
+        target_path_basedir = if target_path.end_with?("/")
+                                target_path
+                              else
+                                File.dirname(target_path)
+                              end
+        FileUtils.mkdir_p(target_path_basedir, mode: @dir_perm)
+
+        log.debug("#{PLUGIN_NAME}: archiving #{file_info.path} to #{target_path}")
+        FileUtils.mv(file_info.path, target_path)
+      rescue StandardError => e
+        log.warn("#{PLUGIN_NAME}: can't archive #{file_info.path} to #{target_path}: #{e}")
       end
     end
   end

data/lib/fluent/plugin/input_static_file/version.rb

@@ -3,7 +3,7 @@
 module Fluent
   module Plugin
     module InputStaticFile
-      VERSION = "0.1.1"
+      VERSION = "0.2.0"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-input-static-file
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Thomas Tych
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-05-30 00:00:00.000000000 Z
+date: 2022-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump
@@ -226,6 +226,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rubocop.yml"
+- ".ruby-version"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile