fluent-plugin-google-cloud-storage-out 0.1.10 → 0.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 13338f63d5b2ddf5e91c56b380d00dec3d24f690
-  data.tar.gz: 46932068452621ea096f14ffcfac24080e102238
+  metadata.gz: 0ae88e22d3d8cac4aa939ccc35fe2af17ca90fbe
+  data.tar.gz: 6e94e55d777c8c0bc07819d2119c68402deffd05
 SHA512:
-  metadata.gz: bd46b63ba61e5447ec224a0753803df242555766330c2882a1d28b2625860cf42b6737918be2ec6cc089dd372bf954cfc5f60ddeca88371b39d74eb8d659092e
-  data.tar.gz: dfcf8bfa3814af0f376ebf44d0d877185aa0aff8a1abed746be3231c14e2f7fe9dc9065059765bc807249d61678d3c3bb7310c68fb8999d17e90a4a7f2f61b15
+  metadata.gz: 2977e856bd530808ec34d62a30c37fcc85ac3c086c4be95a4a88a470611739fa8e1f937daa3ef584eb903a11a1ffda7d7c74b4cb2a00342454cb73e1304ac229
+  data.tar.gz: 4326eaad59d06a09c3416fd2e9095804a5d0fea152176517f8a3027383a31eff0515c5276f0282e78b0fd65f02cdf04cbd2c96b4de7b5b650feb2676d5ba1659

data/fluent-plugin-google-cloud-storage-out.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-google-cloud-storage-out"
-  spec.version       = "0.1.10"
+  spec.version       = "0.1.11"
   spec.authors       = ["Hideki Matsuoka"]
   spec.email         = ["matsuoka.hide@gmail.com"]
 

data/lib/fluent/plugin/out_google_cloud_storage_out.rb

@@ -13,9 +13,17 @@ module Fluent
 
     SUPPORTED_COMPRESS = {
       'gz' => :gz,
-      'gzip' => :gzip,
+      'gzip' => :gz,
     }
 
+    UNIQUE_STRATEGY= {
+      'chunk_id' => :chunk_id,
+      'increment' => :increment,
+      'timestamp' => :timestamp,
+    }
+
+    UNIQUE_PLACE_HOLDER = '${unique}'
+
     #
     # Config Parameters
     #
@@ -40,6 +48,18 @@ module Fluent
     #desc "The tag for out"
     config_param :default_tag, :string, :default => 'tag_missing'
 
+    #desc "The unique strategy for avoid override for same path"
+    config_param :unique_strategy, :default => nil do |val|
+      c = UNIQUE_STRATEGY[val]
+      unless c
+        raise ConfigError, "Unsupported make uniuqe strategy '#{val}'"
+      end
+      c
+    end
+
+    #desc "The format for unique replacement, if you set timestamp to unique_strategy, you should set time format"
+    config_param :unique_format, :string, :default => "%Y%m%d%H%M%S%L"
+
     #desc "Compress flushed file."
     config_param :compress, :default => nil do |val|
       c = SUPPORTED_COMPRESS[val]
@@ -75,9 +95,18 @@ module Fluent
 
       super
 
+      if @path.index(UNIQUE_PLACE_HOLDER).nil? && @unique_strategy
+        raise Fluent::ConfigError, "Path must contain ${unique}, or you set the unique_strategy to nil."
+      end
+
       @formatter = Plugin.new_formatter(@format)
       @formatter.configure(conf)
+      @path_suffix = ".log"
       prepare_client()
+
+      if @unique_strategy == :increment
+         @samepath_counter = 0
+      end
     end
 
     def prepare_client
@@ -103,17 +132,56 @@ module Fluent
       @formatter.format(tag, time, record)
     end
 
-    def path_format(chunk_key)
-      path = Time.strptime(chunk_key, @time_slice_format).strftime(@path)
-      log.debug "GCS Path: #{path}"
-      path
+    def chunk_unique_id_to_str(unique_id)
+      unique_id.unpack('C*').map{|x| x.to_s(16).rjust(2,'0')}.join('')
+    end
+
+    def path_format_with_unique_strategy(path, strategy, chunk_key, chunk_unique)
+       case strategy
+       when nil
+         path
+       when :chunk_id
+         path.gsub(UNIQUE_PLACE_HOLDER, chunk_unique_id_to_str(chunk_unique))
+       when :increment
+         if @before_chunk_key
+           if @before_chunk_key == chunk_key
+             @samepath_counter += 1
+           else
+             @samepath_counter = 0
+           end
+         end
+         @before_chunk_key = chunk_key
+         path.gsub(UNIQUE_PLACE_HOLDER, "#{@samepath_counter}")
+       when :timestamp
+         path.gsub(UNIQUE_PLACE_HOLDER, Time.now.strftime(@unique_format))
+       end
+    end
+
+    def path_format(chunk)
+      # format from chunk key
+      path = Time.strptime(chunk.key, @time_slice_format).strftime(@path)
+
+      # format for make unique
+      path = path_format_with_unique_strategy(path, @unique_strategy, chunk.key, chunk.unique_id)
+
+      # append .log
+      unless path.include?(".log")
+        path.concat(@path_suffix)
+      end
+
+      # append .gz
+      case @compress
+      when nil
+        path
+      when :gz
+        "#{path}.gz"
+      end
     end
 
     def send(path, data)
       mimetype = MIME::Types.type_for(path).first
-
       io = nil
-      if SUPPORTED_COMPRESS.include?(@compress)
+      if @compress
         io = StringIO.new("")
         writer = Zlib::GzipWriter.new(io)
         writer.write(data)
@@ -127,10 +195,9 @@ module Fluent
     end
 
     def write(chunk)
-      gcs_path = path_format(chunk.key)
+      gcs_path = path_format(chunk)
       send(gcs_path, chunk.read)
       gcs_path
     end
   end
 end
-

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud-storage-out
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 0.1.11
 platform: ruby
 authors:
 - Hideki Matsuoka