fluent-plugin-google-cloud 0.5.3.grpc.alpha.2 → 0.5.3.grpc.alpha.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d313f90fa86ac062a46a65bd400a8729085f4c3f
-  data.tar.gz: 6a699b31b50a807178ebd97a913e47666c0f34b2
+  metadata.gz: cdae7652026e7ff5ad8ac5593c6ecf016948f41f
+  data.tar.gz: 85162c0a5db2c340faf21e60256a8c3fa66be30c
 SHA512:
-  metadata.gz: fc3b4b13b78bf6c4d04976e79fd91f96801c85e0d26c7ffff93fcb4cc9e27c6d741626f2cd2bf89aeb53932d739568c05e0d3549a7746eec0b3b3ed97d22ab60
-  data.tar.gz: 9edc718d851cb46e511743f16f0228167aafefd0e341a23b4619d57a95ec680e2d17025e7c5b146d435d96a4d79ebb00f4ea7f626e9c12da11d1a5f615486695
+  metadata.gz: bdc518fb676f756a572d16d0589ab7297bf2d67a78aac6ffb1772dafa0c2025ad33641c8290ac11c6d467704ab4bcb92b59bf6dc65cff2d23939636e470f88be
+  data.tar.gz: a209e99c4390c0661438cb24eeaa9628615743096d0021e001a327eac0a4ba2e1abc573adabfa98d0ba0e78ad779521ff046adcb3570e747037bd47b704481c2

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.5.3.grpc.alpha.2)
+    fluent-plugin-google-cloud (0.5.3.grpc.alpha.3)
       fluentd (~> 0.10, <= 0.13)
       google-api-client (> 0.9)
       googleapis-common-protos (~> 1.3)
@@ -12,7 +12,8 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.4.0)
+    addressable (2.5.0)
+      public_suffix (~> 2.0, >= 2.0.2)
     ast (2.3.0)
     astrolabe (1.3.1)
       parser (~> 2.2)
@@ -21,7 +22,7 @@ GEM
       safe_yaml (~> 1.0.0)
     faraday (0.9.2)
       multipart-post (>= 1.2, < 3)
-    fluentd (0.12.26)
+    fluentd (0.12.29)
       cool.io (>= 1.2.2, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
       json (>= 1.4.3)
@@ -31,7 +32,7 @@ GEM
       tzinfo (>= 1.0.0)
       tzinfo-data (>= 1.0.0)
       yajl-ruby (~> 1.0)
-    google-api-client (0.9.13)
+    google-api-client (0.9.20)
       addressable (~> 2.3)
       googleauth (~> 0.5)
       httpclient (~> 2.7)
@@ -40,8 +41,8 @@ GEM
       mime-types (>= 1.6)
       representable (~> 2.3.0)
       retriable (~> 2.0)
-    google-protobuf (3.0.0)
-    googleapis-common-protos (1.3.0)
+    google-protobuf (3.0.2)
+    googleapis-common-protos (1.3.4)
       google-protobuf (~> 3.0)
       grpc (~> 1.0)
     googleauth (0.5.1)
@@ -52,15 +53,15 @@ GEM
       multi_json (~> 1.11)
       os (~> 0.9)
       signet (~> 0.7)
-    grpc (1.0.0)
-      google-protobuf (~> 3.0)
+    grpc (1.0.1)
+      google-protobuf (~> 3.0.2)
       googleauth (~> 0.5.1)
     hashdiff (0.3.0)
     http_parser.rb (0.6.0)
-    httpclient (2.8.2.3)
+    httpclient (2.8.2.4)
     hurley (0.2)
     json (1.8.3)
-    jwt (1.5.4)
+    jwt (1.5.6)
     little-plugger (1.1.4)
     logging (2.1.0)
       little-plugger (~> 1.1)
@@ -70,16 +71,17 @@ GEM
     mime-types (3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2016.0521)
-    mocha (1.1.0)
+    mocha (1.2.1)
       metaclass (~> 0.0.1)
     msgpack (0.5.12)
     multi_json (1.12.1)
     multipart-post (2.0.0)
     os (0.9.6)
-    parser (2.3.1.2)
+    parser (2.3.1.4)
       ast (~> 2.2)
-    power_assert (0.3.0)
+    power_assert (0.3.1)
     powerpack (0.1.1)
+    public_suffix (2.0.4)
     rainbow (2.1.0)
     rake (10.5.0)
     representable (2.3.0)
@@ -100,19 +102,19 @@ GEM
       jwt (~> 1.5)
       multi_json (~> 1.10)
     string-scrub (0.0.5)
-    test-unit (3.2.0)
+    test-unit (3.2.2)
       power_assert
     thread_safe (0.3.5)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
-    tzinfo-data (1.2016.6)
+    tzinfo-data (1.2016.9)
       tzinfo (>= 1.0.0)
     uber (0.0.15)
     webmock (1.24.6)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff
-    yajl-ruby (1.2.1)
+    yajl-ruby (1.3.0)
 
 PLATFORMS
   ruby
@@ -126,4 +128,4 @@ DEPENDENCIES
   webmock (~> 1.17)
 
 BUNDLED WITH
-   1.12.5
+   1.13.6

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      = \
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.5.3.grpc.alpha.2'
+  gem.version       = '0.5.3.grpc.alpha.3'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')

data/lib/fluent/plugin/out_google_cloud.rb

@@ -30,7 +30,7 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'
-    PLUGIN_VERSION = '0.5.3.grpc.alpha.1'
+    PLUGIN_VERSION = '0.5.3.grpc.alpha.3'
 
     # Constants for service names.
     APPENGINE_SERVICE = 'appengine.googleapis.com'
@@ -122,6 +122,16 @@ module Fluent
     # Cloud Logging API.
     config_param :use_grpc, :bool, :default => false
 
+    # Whether to allow non-UTF-8 characters in user logs. If set to true, any
+    # non-UTF-8 character would be replaced by the string specified by
+    # 'non_utf8_replacement_string'. If set to false, any non-UTF-8 character
+    # would trigger the plugin to error out.
+    config_param :coerce_to_utf8, :bool, :default => true
+
+    # If 'coerce_to_utf8' is set to true, any non-UTF-8 character would be
+    # replaced by the string specified here.
+    config_param :non_utf8_replacement_string, :string, :default => ' '
+
     # DEPRECATED: The following parameters, if present in the config
     # indicate that the plugin configuration must be updated.
     config_param :auth_method, :string, :default => nil
@@ -363,9 +373,9 @@ module Fluent
           if @use_grpc
             entry = Google::Logging::V1::LogEntry.new(
               metadata: Google::Logging::V1::LogEntryMetadata.new(
-                service_name: @service_name.encode('utf-8'),
-                project_id: @project_id.encode('utf-8'),
-                zone: @zone.encode('utf-8'),
+                service_name: convert_to_utf8(@service_name),
+                project_id: convert_to_utf8(@project_id),
+                zone: convert_to_utf8(@zone),
                 labels: {}
               ))
           else
@@ -476,9 +486,9 @@ module Fluent
             client = api_client
 
             labels_utf8_pairs = labels.map do |k, v|
-              [k.encode('utf-8'), v.encode('utf-8')]
+              [k.encode('utf-8'), convert_to_utf8(v)]
             end
-            utf8_log_name = log_name.encode('utf-8')
+            utf8_log_name = convert_to_utf8(log_name)
 
             write_request = Google::Logging::V1::WriteLogEntriesRequest.new(
               log_name: "projects/#{@project_id}/logs/#{utf8_log_name}",
@@ -1308,7 +1318,7 @@ module Fluent
       when Numeric
         ret.number_value = value
       when String
-        ret.string_value = value.encode('utf-8')
+        ret.string_value = convert_to_utf8(value)
       when TrueClass
         ret.bool_value = true
       when FalseClass
@@ -1352,15 +1362,16 @@ module Fluent
       # 2. This is an unstructured Container log and the 'log' key is available
       # 3. The only remaining key is 'message'
       if @service_name == CLOUDFUNCTIONS_SERVICE && @cloudfunctions_log_match
-        entry.text_payload = @cloudfunctions_log_match['text']
+        entry.text_payload = convert_to_utf8(
+          @cloudfunctions_log_match['text'])
       elsif @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
-        entry.text_payload = record['log']
+        entry.text_payload = convert_to_utf8(record['log'])
       elsif is_json
         entry.struct_payload = struct_from_ruby(record)
       elsif @service_name == CONTAINER_SERVICE && record.key?('log')
-        entry.text_payload = record['log']
+        entry.text_payload = convert_to_utf8(record['log'])
       elsif record.size == 1 && record.key?('message')
-        entry.text_payload = record['message']
+        entry.text_payload = convert_to_utf8(record['message'])
       else
         entry.struct_payload = struct_from_ruby(record)
       end
@@ -1415,5 +1426,29 @@ module Fluent
       end
       @client
     end
+
+    # Encode as UTF-8. If 'coerce_to_utf8' is set to true in the config, any
+    # non-UTF-8 character would be replaced by the string specified by
+    # 'non_utf8_replacement_string'. If 'coerce_to_utf8' is set to false, any
+    # non-UTF-8 character would trigger the plugin to error out.
+    def convert_to_utf8(input)
+      if @coerce_to_utf8
+        input.encode(
+          'utf-8',
+          invalid: :replace,
+          undef: :replace,
+          replace: @non_utf8_replacement_string)
+      else
+        begin
+          input.encode('utf-8')
+        rescue EncodingError
+          @log.error 'Encountered encoding issues potentially due to non ' \
+                     'UTF-8 characters. To allow non-UTF-8 characters and ' \
+                     'replace them with spaces, please set "coerce_to_utf8" ' \
+                     'to true.'
+          raise
+        end
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.5.3.grpc.alpha.2
+  version: 0.5.3.grpc.alpha.3
 platform: ruby
 authors:
 - Todd Derr
@@ -9,124 +9,124 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-07 00:00:00.000000000 Z
+date: 2016-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0.10'
-    - - "<="
+    - - <=
       - !ruby/object:Gem::Version
         version: '0.13'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0.10'
-    - - "<="
+    - - <=
       - !ruby/object:Gem::Version
         version: '0.13'
 - !ruby/object:Gem::Dependency
   name: googleapis-common-protos
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: google-api-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - '>'
       - !ruby/object:Gem::Version
         version: '0.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - '>'
       - !ruby/object:Gem::Version
         version: '0.9'
 - !ruby/object:Gem::Dependency
   name: googleauth
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0.4'
 - !ruby/object:Gem::Dependency
   name: grpc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.8'
 - !ruby/object:Gem::Dependency
   name: mocha
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '10.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '10.3'
 - !ruby/object:Gem::Dependency
@@ -147,28 +147,28 @@ dependencies:
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.17'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.17'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '3.0'
 description: |2
@@ -211,12 +211,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - '>'
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []