fluent-plugin-google-cloud-storage-out 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6fbb34bba0df7e05a225f4731482ed6de883b937
-  data.tar.gz: ee06275353587a4f67c831bbd14eeff796b9aea3
+  metadata.gz: 04a0022a7b4694f01182f15dbdba737a4d2bd006
+  data.tar.gz: 99da050af82790baa08dcc5d35e5702498009651
 SHA512:
-  metadata.gz: b4c11a24ced090eb47b589743afd77dd95f4bc577f912f0eeb507fc8830c58ba96d38bb911118d07c7638ebe9e44254abc0db70da504deefb5456e37d72d75b5
-  data.tar.gz: 8e4563e54eba107f13aeb7ed758749154c1b2b8289bb2b9cc8ff01ace93421659215cf5387c3f373c60f2315070ca4ae96f4b3f102cc1ec28b03caee4a92e372
+  metadata.gz: 8d604a2370b3903d940be02d21439612c9dd762934f4d408550581fae2c6baafae0a2fa12a71125363dc9570e0ca64a2df1c662fd91e282f8151382352212e9f
+  data.tar.gz: c326ce6e5db73f466a5187e46b05f4a46f3c8bc31d6e1a60eafb29125e2d6c77e0fb22d75b7c78300fabd2c5ead556d24c085006e78d1b62160a8fc13653f8c3

data/lib/fluent/plugin/out_google_cloud_storage_out.rb

@@ -1,163 +1,147 @@
 # -*- coding: utf-8 -*-
 
-require "fluent/mixin/config_placeholders"
-require "fluent/log"
-
-require 'googleauth'
-require 'google/apis/storage_v1'
-
-require 'fluent/plugin/version'
-
-class Fluent::GoogleCloudStorageOut < Fluent::TimeSlicedOutput
-  Storage = Google::Apis::StorageV1
-  ServiceAccountCredentials = Google::Auth::ServiceAccountCredentials
-
-  extend GoogleCloudStorageOut
-
-  Fluent::Plugin.register_output('google_cloud_storage_out', self)
-
-  SUPPORTED_COMPRESS = {
-    'gz' => :gz,
-    'gzip' => :gzip,
-  }
-
-  config_set_default :buffer_type, 'file'
-  config_set_default :time_slice_format, '%Y%m%d'
-  config_set_default :flush_interval, nil
-
-  include Fluent::Mixin::ConfigPlaceholders
+module Fluent
+  require "fluent/log"
+  require "fluent/mixin/config_placeholders"
+  require 'googleauth'
+  require 'google/apis/storage_v1'
+  require 'fluent/plugin/version'
+  class GoogleCloudStorageOut < TimeSlicedOutput
+    Plugin.register_output('google_cloud_storage_out', self)
+
+    Storage = Google::Apis::StorageV1
+    ServiceAccountCredentials = Google::Auth::ServiceAccountCredentials
+
+    extend GoogleCloudStorageOut
+
+    SUPPORTED_COMPRESS = {
+      'gz' => :gz,
+      'gzip' => :gzip,
+    }
+
+    #
+    # Config Parameters
+    #
+    config_set_default :buffer_type, 'file'
+    config_set_default :time_slice_format, '%Y%m%d'
+    config_set_default :flush_interval, nil
+
+    include Mixin::ConfigPlaceholders
+
+    desc "The path of Service Account Json key."
+    config_param :service_account_json_key_path, :string
+
+    desc "The bucket ID for destination for store."
+    config_param :bucket_id, :string
+
+    desc "The directory path for store."
+    config_param :path, :string
+
+    desc "The format of the file content. The default is out_file"
+    config_param :format, :string, :default => 'out_file'
+
+    desc "The tag for out"
+    config_param :default_tag, :string, :default => 'tag_missing'
+
+    desc "Compress flushed file."
+    config_param :compress, :default => nil do |val|
+      c = SUPPORTED_COMPRESS[val]
+      unless c
+        raise ConfigError, "Unsupported compression algorithm '#{val}'"
+      end
+      c
+    end
 
-  desc "The path of Service Account Json key."
-  config_param :service_account_json_key_path, :string
+    def initialize
+      super
+      require 'zlib'
+      require 'net/http'
+      require 'time'
+      require 'mime-types'
+    end
 
-  desc "The bucket ID for destination for store."
-  config_param :bucket_id, :string
+    # Define `log` method for v0.10.42 or earlier
+    unless method_defined?(:log)
+      define_method("log") { $log }
+    end
 
-  desc "The directory path for store."
-  config_param :path, :string
+    def configure(conf)
+      if conf['path']
+        if conf['path'].index('%S')
+          conf['time_slice_format'] = '%Y%m%d%H%M%S'
+        elsif conf['path'].index('%M')
+          conf['time_slice_format'] = '%Y%m%d%H%M'
+        elsif conf['path'].index('%H')
+          conf['time_slice_format'] = '%Y%m%d%H'
+        end
+      end
 
-  desc "The format of the file content. The default is out_file"
-  config_param :format, :string, :default => 'out_file'
+      super
 
-  desc "Compress flushed file."
-  config_param :compress, :default => nil do |val|
-    c = SUPPORTED_COMPRESS[val]
-    unless c
-      raise ConfigError, "Unsupported compression algorithm '#{val}'"
+      @formatter = Plugin.new_formatter(@format)
+      @formatter.configure(conf)
+      prepare_client()
     end
-    c
-  end
 
-  include Fluent::Mixin::PlainTextFormatter
+    def prepare_client
 
-  config_param :default_tag, :string, :default => 'tag_missing'
+      storage = Storage::StorageService.new
+      scopes = [Storage::AUTH_CLOUD_PLATFORM, Storage::AUTH_DEVSTORAGE_FULL_CONTROL]
+      storage.authorization = ServiceAccountCredentials.make_creds(
+        {
+          :json_key_io => File.open(@service_account_json_key),
+          :scope => scopes
+        }
+      )
 
-  def initialize
-    super
-    require 'zlib'
-    require 'net/http'
-    require 'time'
-    require 'mime-types'
-  end
-
-  # Define `log` method for v0.10.42 or earlier
-  unless method_defined?(:log)
-    define_method("log") { $log }
-  end
-
-  #def call_google_api(params)
-  #  # refresh_auth
-  #  if @google_api_client.authorization.expired?
-  #    @google_api_client.authorization.fetch_access_token!
-  #  end
-  #  return @google_api_client.execute(params)
-  #end
-
-  def configure(conf)
-    if conf['path']
-      if conf['path'].index('%S')
-        conf['time_slice_format'] = '%Y%m%d%H%M%S'
-      elsif conf['path'].index('%M')
-        conf['time_slice_format'] = '%Y%m%d%H%M'
-      elsif conf['path'].index('%H')
-        conf['time_slice_format'] = '%Y%m%d%H'
-      end
+      @google_api_client = storage
     end
 
-    super
-
-    @formatter = Plugin.new_formatter(@format)
-    @formatter.configure(conf)
-    prepare_client()
-  end
-
-  def prepare_client
+    def start
+      super
+    end
 
-    storage = Storage::StorageService.new
-    scopes = [Storage::AUTH_CLOUD_PLATFORM, Storage::AUTH_DEVSTORAGE_FULL_CONTROL]
-    storage.authorization = ServiceAccountCredentials.make_creds(
-      {
-        :json_key_io => File.open(@service_account_json_key),
-        :scope => scopes
-      }
-    )
+    def shutdown
+      super
+    end
 
-    @google_api_client = storage
-  end
+    def format(tag, time, record)
+      @formatter.format(tag, time, record)
+    end
 
-  def start
-    super
-  end
+    def path_format(chunk_key)
+      path = Time.strptime(chunk_key, @time_slice_format).strftime(@path)
+      log.debug "GCS Path: #{path}"
+      path
+    end
 
-  def shutdown
-    super
-  end
+    def send(path, data)
+      mimetype = MIME::Types.type_for(path).first
+
+      io = nil
+      if SUPPORTED_COMPRESS.include?(@compress)
+        io = StringIO.new("")
+        writer = Zlib::GzipWriter.new(io)
+        writer.write(data)
+        writer.finish
+        io.rewind
+      else
+        io = StringIO.new(data)
+      end
 
-  def format(tag, time, record)
-    @formatter.format(tag, time, record)
-  end
+      media = Google::APIClient::UploadIO.new(io, mimetype.content_type, File.basename(path))
 
-  def path_format(chunk_key)
-    path = Time.strptime(chunk_key, @time_slice_format).strftime(@path)
-    log.debug "GCS Path: #{path}"
-    path
-  end
+      @gogle_api_client.insert_object(@bucket_id, upload_source: io, name: path, content_type:mimetype_content_type)
 
-  def send(path, data)
-    mimetype = MIME::Types.type_for(path).first
-
-    io = nil
-    if SUPPORTED_COMPRESS.include?(@compress)
-      io = StringIO.new("")
-      writer = Zlib::GzipWriter.new(io)
-      writer.write(data)
-      writer.finish
-      io.rewind
-    else
-      io = StringIO.new(data)
     end
 
-    media = Google::APIClient::UploadIO.new(io, mimetype.content_type, File.basename(path))
+    def write(chunk)
+      gcs_path = path_format(chunk.key)
 
-    @gogle_api_client.insert_object(@bucket_id, upload_source: io, name: path, content_type:mimetype_content_type)
+      send(gcs_path, chunk.read)
 
-    #call_google_api(api_method: @storage_api.objects.insert,
-    #                parameters: {
-    #                  uploadType: "multipart",
-    #                  project: @project_id,
-    #                  bucket: @bucket_id,
-    #                  name: path
-    #                },
-    #                body_object: { contentType: media.content_type },
-    #                media: media)
-  end
-
-  def write(chunk)
-    gcs_path = path_format(chunk.key)
-
-    send(gcs_path, chunk.read)
-
-    gcs_path
+      gcs_path
+    end
   end
 end
 

data/lib/fluent/plugin/version.rb

@@ -1,3 +1,3 @@
 module GoogleCloudStorageOut
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud-storage-out
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Hideki Matsuoka