fluent-plugin-google-cloud-storage 0.3.3 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 64b3a5589fde0af5eb5aeab523383f157893002a
-  data.tar.gz: 8eeb8cb19a978ada03adb491ffc8b6241cf23027
+  metadata.gz: bf9c112b4b4d08e0773ae2ae0793e2b8d7b765a1
+  data.tar.gz: 36e41506714e22dbdae2f01554503787c6506efe
 SHA512:
-  metadata.gz: bdd755f5d5d6c8ee767c4530f3ff1db3a3f75a5cd779557a536e54782c639edda7eaf3aee00459026634580326ed97e8743dfe3e1e3c596fe8b1d184bc85ceec
-  data.tar.gz: 697f1f5cc9f03cae9e77b084bf00bcc6dcfb92f47113b1376197b989fcd57fdaceda3ebfa8734c48a16248e4eef06cc81399dad8bd50ef799baef430c653cd6f
+  metadata.gz: a5115e5522a4bad1399cc0188e47967f20c9f48d59ed2f8fd5b4bbc03544a0e94fac6166229e7b61d7008bb5b55b4dbe782a8ae19cab74e0077b1a20aab2e7be
+  data.tar.gz: 710eff5033c5f9e3c14f8a574c4d04ecf618c90ba7de800e164c956c2a9febf38052499ce7d75a73c0bee7cac71a9d6b3768c35160bae1593476e28bbbc9b032

data/README.md

@@ -17,9 +17,39 @@ And you can specify output file path as 'path path/to/dir/access.%Y%m%d.log', th
 
 ## Configuration
 
-### GoogleCloudStorageOutput
+### Examples
 
-To store data by time,tag,json (same with 'type file') over WebHDFS:
+#### Complete Example
+
+    # tail
+    <source>
+      type tail
+      format none
+      path /tmp/test.log
+      pos_file /var/log/td-agent/test.pos
+      tag tail.test
+    </source>
+
+    # post to GCS
+    <match tail.test>
+      type google_cloud_storage
+      service_email xxx.xxx.com
+      service_pkcs12_path /etc/td-agent/My_First_Project-xxx.p12
+      project_id handy-compass-xxx
+      bucket_id test_bucket
+      path tail.test/%Y/%m/%d/%H/${hostname}/${chunk_id}.log.gz
+      output_include_time false
+      output_include_tag  false
+      buffer_path /var/log/td-agent/buffer/tail.test
+      # flush_interval 600s
+      buffer_chunk_limit 128m
+      time_slice_wait 300s
+      compress gzip
+    </match>
+
+#### More Examples
+
+To store data by `time,tag,json` (same with 'type file') with GCS:
 
     <match access.**>
       type google_cloud_storage
@@ -27,7 +57,7 @@ To store data by time,tag,json (same with 'type file') over WebHDFS:
       service_pkcs12_path /path/to/key.p12
       project_id name-of-project
       bucket_id name-of-bucket
-      path path/to/access.%Y%m%d_%H.log
+      path path/to/access.%Y%m%d_%H.${chunk_id}.log
     </match>
 
 To specify the pkcs12 file's password, use `service_pkcs12_password`:
@@ -39,7 +69,7 @@ To specify the pkcs12 file's password, use `service_pkcs12_password`:
       service_pkcs12_password SECRET_PASSWORD
       project_id name-of-project
       bucket_id name-of-bucket
-      path path/to/access.%Y%m%d_%H.log
+      path path/to/access.%Y%m%d_%H.${chunk_id}.log
     </match>
 
 If you want JSON object only (without time or tag or both on header of lines), specify it by `output_include_time` or `output_include_tag` (default true):
@@ -50,7 +80,7 @@ If you want JSON object only (without time or tag or both on header of lines), s
       service_pkcs12_path /path/to/key.p12
       project_id name-of-project
       bucket_id name-of-bucket
-      path path/to/access.%Y%m%d_%H.log
+      path path/to/access.%Y%m%d_%H.${chunk_id}.log
       output_include_time false
       output_include_tag  false
     </match>
@@ -88,18 +118,22 @@ To store data compressed (gzip only now):
       compress gzip
     </match>
 
-### Performance notifications
+### Major Caveat
 
 As GCS does not support appending to files, if you have multiple fluentd nodes,
-you most likely want to log to multiple files. You can use '${hostname}' or
+you most likely each to log to separate files. You can use '${hostname}' or
 '${uuid:random}' placeholders in configuration for this purpose.
 
+Note the `${chunk_id}` placeholder in the following paths. The plugin requires the presence
+of the placeholder to guarantee that each flush will not overwrite an existing
+file.
+
 For hostname:
 
     <match access.**>
       type google_cloud_storage
       # ...
-      path log/access/%Y%m%d/${hostname}.log
+      path log/access/%Y%m%d/${hostname}.${chunk_id}.log
     </match>
 
 Or with random filename (to avoid duplicated file name only):
@@ -107,10 +141,11 @@ Or with random filename (to avoid duplicated file name only):
     <match access.**>
       type google_cloud_storage
       # ...
-      path log/access/%Y%m%d/${uuid:random}.log
+      path log/access/%Y%m%d/${uuid:random}.${chunk_id}.log
     </match>
 
-With configurations above, you can handle all of files of '/log/access/20120820/*' as specified timeslice access logs.
+With the configurations above, you can handle all of files of
+'/log/access/20120820/*' as specified timeslice access logs.
 
 ## TODO
 

data/fluent-plugin-google-cloud-storage.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-google-cloud-storage"
-  gem.version       = "0.3.3"
+  gem.version       = "1.0.0"
   gem.authors       = ["Hsiu-Fan Wang"]
   gem.email         = ["hfwang@porkbuns.net"]
   gem.summary       = %q{Fluentd plugin to write data to Google Cloud Storage}
@@ -19,5 +19,6 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "fluentd", '>= 0.10.53'
   gem.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'
   gem.add_runtime_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
-  gem.add_runtime_dependency "google-api-client", '~> 0.7'
+  gem.add_runtime_dependency "google-api-client", '0.8.6'
+  gem.add_runtime_dependency "mime-types", '>= 3.0'
 end

data/lib/fluent/plugin/out_google_cloud_storage.rb

@@ -2,6 +2,7 @@
 
 require 'fluent/mixin/config_placeholders'
 require 'fluent/mixin/plaintextformatter'
+require 'fluent/log'
 
 class Fluent::GoogleCloudStorageOutput < Fluent::TimeSlicedOutput
   Fluent::Plugin.register_output('google_cloud_storage', self)
@@ -45,6 +46,7 @@ class Fluent::GoogleCloudStorageOutput < Fluent::TimeSlicedOutput
     require 'time'
     require 'google/api_client'
     require 'signet/oauth_2/client'
+    require 'mime-types'
   end
 
   # Define `log` method for v0.10.42 or earlier
@@ -110,7 +112,9 @@ class Fluent::GoogleCloudStorageOutput < Fluent::TimeSlicedOutput
   end
 
   def path_format(chunk_key)
-    Time.strptime(chunk_key, @time_slice_format).strftime(@path)
+    path = Time.strptime(chunk_key, @time_slice_format).strftime(@path)
+    log.debug "GCS Path: #{path}"
+    path
   end
 
   def chunk_unique_id_to_str(unique_id)
@@ -118,7 +122,7 @@ class Fluent::GoogleCloudStorageOutput < Fluent::TimeSlicedOutput
   end
 
   def send_data(path, data)
-    content_type = "application/json"
+    mimetype = MIME::Types.type_for(path).first
 
     io = nil
     if ["gz", "gzip"].include?(@compress)
@@ -131,7 +135,7 @@ class Fluent::GoogleCloudStorageOutput < Fluent::TimeSlicedOutput
       io = StringIO.new(data)
     end
 
-    media = Google::APIClient::UploadIO.new(io, content_type, File.basename(path))
+    media = Google::APIClient::UploadIO.new(io, mimetype.content_type, File.basename(path))
 
     call_google_api(api_method: @storage_api.objects.insert,
                     parameters: {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud-storage
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 1.0.0
 platform: ruby
 authors:
 - Hsiu-Fan Wang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-24 00:00:00.000000000 Z
+date: 2016-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -70,16 +70,30 @@ dependencies:
   name: google-api-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: 0.8.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: 0.8.6
+- !ruby/object:Gem::Dependency
+  name: mime-types
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
 description: Google Cloud Storage fluentd output
 email:
 - hfwang@porkbuns.net
@@ -117,10 +131,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to write data to Google Cloud Storage
 test_files:
 - test/helper.rb
 - test/plugin/test_out_google_cloud_storage.rb
+has_rdoc: