fluent-plugin-input-static-file 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4859ef4b60d4298e26fe8ad4aa30f5d9adc3ee11
-  data.tar.gz: d9aca614c7c03071409094f0a6b234808f82a46a
+  metadata.gz: 5bdfe4c08f457eded785a80862beb1104cf8fd95
+  data.tar.gz: b8ccfdb81d8ad125d346fdc50824e35640d50fc1
 SHA512:
-  metadata.gz: c85bdcec33cd1f65f6867ebc2b6f881ed98f5e9d8f9160df5e9370b9e1f7b890c5c50cbd87eb5fc61b1185a967bd1d84b08413b99502d7e8c9a258239bfedf23
-  data.tar.gz: 6e0f362a15d9844f7ed60efce3901c064e5e7ba41881b05d26440ad9a217ba9ce45ed967f60a3b6152b672fcfef39ba6e622bb89bfab00b6b27096b9197234e7
+  metadata.gz: 206998e202d008ca737b018457afa8fff1ea9e50a994cd172a0e23f4b4cfc521707894ca786a2be519d31bcd18595ae9e4d8dbb4cba7a14d6460bf04fbd6418a
+  data.tar.gz: ccbbb8af45d63974d19de472c8d3137a17d743596fda76a5f2d0b2dfbcf0dd74952b294c73da50f3ae431f21b656187716924e6ab3d8892906387fbf9ea016c9

data/.ruby-version

@@ -0,0 +1 @@
+2.4.10

data/lib/fluent/plugin/in_static_file/file_tracker.rb

@@ -77,7 +77,7 @@ module Fluent
 
             @file.pos = 0
             @file.truncate(0)
-            @file.write(@cache.values.maps(&:to_file_tracker_entry_format).join)
+            @file.write(@cache.values.map(&:to_file_tracker_entry_format).join)
           end
         end
       end

data/lib/fluent/plugin/in_static_file.rb

@@ -3,7 +3,7 @@
 require "fluent/plugin/input"
 require "fluent/config/error"
 
-require "lib/fluent/plugin/in_static_file/file_tracker"
+require "fluent/plugin/in_static_file/file_tracker"
 
 if Fluent.windows?
   require "fluent/plugin/file_wrapper"
@@ -15,14 +15,14 @@ module Fluent
   module Plugin
     # InStaticFile is an input plugin for file with static content
     class InStaticFile < Fluent::Plugin::Input
-      Fluent::Plugin.register_input("static_file", self)
+      PLUGIN_NAME = "static_file"
+
+      Fluent::Plugin.register_input(PLUGIN_NAME, self)
 
       helpers :timer, :compat_parameters, :parser
 
       RESERVED_CHARS = ["/", "*", "%"].freeze
 
-      # FIXME: path_key
-
       desc "The paths to read. Multiple paths can be specified, separated by comma."
       config_param :path, :string
       desc "path delimiter used for spliting path config"
@@ -40,12 +40,17 @@ module Fluent
 
       desc "The tag of the event."
       config_param :tag, :string
+      desc "Add the log path being tailed to records. Specify the field name to be used."
+      config_param :path_key, :string, default: nil
 
       desc "Fluentd will record the position it last read into this file."
       config_param :pos_file, :string, default: nil
       desc "Follow inodes instead of following file names. Guarantees more stable delivery and allows to use * in path pattern with rotating files"
       config_param :follow_inodes, :bool, default: false
 
+      desc "When processed move to another location"
+      config_param :archive_to, :string, default: nil
+
       def initialize
         super
 
@@ -70,19 +75,22 @@ module Fluent
         end
 
         @paths = @path.split(@path_delimiter).map(&:strip).uniq
-        raise Fluent::ConfigError, "static_file: 'path' parameter is required on static_file input" if @paths.empty?
+        if @paths.empty?
+          raise Fluent::ConfigError,
+                "#{PLUGIN_NAME}: 'path' parameter is required on #{PLUGIN_NAME} input"
+        end
 
         if @pos_file
           if @variable_store.key?(@pos_file) && !called_in_test?
             plugin_id_using_this_path = @variable_store[@pos_file]
             raise Fluent::ConfigError,
-                  "Other 'in_static_file' plugin already use same pos_file path: plugin_id = #{plugin_id_using_this_path}, pos_file path = #{@pos_file}"
+                  "Other '#{PLUGIN_NAME}' plugin already use same pos_file path: plugin_id = #{plugin_id_using_this_path}, pos_file path = #{@pos_file}"
           end
           @variable_store[@pos_file] = plugin_id
         else
           raise Fluent::ConfigError, "Can't follow inodes without pos_file configuration parameter" if @follow_inodes
 
-          log.warn "'pos_file PATH' parameter is not set to a 'static_file' source."
+          log.warn "'pos_file PATH' parameter is not set to a '#{PLUGIN_NAME}' source."
           log.warn "this parameter is highly recommended to save the file status."
         end
 
@@ -188,31 +196,56 @@ module Fluent
       end
 
       def untrack_files(files_info)
-        log.debug("static_file: untrack files: #{files_info.keys}")
+        log.debug("#{PLUGIN_NAME}: untrack files: #{files_info.keys}")
         files_info.each do |_id, file_info|
           @file_tracker.remove(file_info)
         end
       end
 
       def process_files(files_info)
-        log.debug("static_file: process files: #{files_info.keys}")
+        log.debug("#{PLUGIN_NAME}: process files: #{files_info.keys}")
         files_info.each do |_id, file_info|
           process_file(file_info)
         end
       end
 
       def process_file(file_info)
-        log.debug("static_file: process file: #{file_info.path}")
+        log.debug("#{PLUGIN_NAME}: process file: #{file_info.path}")
 
         return if @file_tracker.has?(file_info)
 
         File.open(file_info.path, "rb") do |f|
           @parser.parse(f) do |time, record|
+            record[@path_key] ||= file_info.path unless @path_key.nil?
             router.emit(tag, time, record)
           end
         end
 
         @file_tracker.add(file_info)
+
+        archive(file_info)
+      rescue StandardError => e
+        log.error "#{PLUGIN_NAME}: trying to process unparsable file #{file_info.path}: #{e}"
+        log.debug_backtrace(e.backtrace)
+      end
+
+      def archive(file_info)
+        return if @archive_to.nil?
+
+        file_name = File.basename(file_info.path)
+        target_path = format(@archive_to, file_name)
+
+        target_path_basedir = if target_path.end_with?("/")
+                                target_path
+                              else
+                                File.dirname(target_path)
+                              end
+        FileUtils.mkdir_p(target_path_basedir, mode: @dir_perm)
+
+        log.debug("#{PLUGIN_NAME}: archiving #{file_info.path} to #{target_path}")
+        FileUtils.mv(file_info.path, target_path)
+      rescue StandardError => e
+        log.warn("#{PLUGIN_NAME}: can't archive #{file_info.path} to #{target_path}: #{e}")
       end
     end
   end

data/lib/fluent/plugin/input_static_file/version.rb

@@ -3,7 +3,7 @@
 module Fluent
   module Plugin
     module InputStaticFile
-      VERSION = "0.1.0"
+      VERSION = "0.2.1"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-input-static-file
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Thomas Tych
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-05-30 00:00:00.000000000 Z
+date: 2022-06-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump
@@ -226,6 +226,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rubocop.yml"
+- ".ruby-version"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile