fluent-plugin-google-cloud 0.4.1 → 0.4.2

data/Gemfile.lock

@@ -1,16 +1,16 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.3.1)
+    fluent-plugin-google-cloud (0.4.1)
       fluentd (>= 0.10)
       google-api-client (>= 0.8)
       googleauth (~> 0.4)
-      json (~> 1.8.3)
+      json (~> 1.8.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (4.2.1)
+    activesupport (4.2.3)
       i18n (~> 0.7)
       json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
@@ -27,7 +27,7 @@ GEM
     extlib (0.9.16)
     faraday (0.9.1)
       multipart-post (>= 1.2, < 3)
-    fluentd (0.12.11)
+    fluentd (0.12.12)
       cool.io (>= 1.2.2, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
       json (>= 1.4.3)
@@ -58,7 +58,7 @@ GEM
     http_parser.rb (0.6.0)
     i18n (0.7.0)
     json (1.8.3)
-    jwt (1.5.0)
+    jwt (1.5.1)
     launchy (2.4.3)
       addressable (~> 2.3)
     little-plugger (1.1.3)
@@ -90,7 +90,7 @@ GEM
     thread_safe (0.3.5)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
-    tzinfo-data (1.2015.4)
+    tzinfo-data (1.2015.5)
       tzinfo (>= 1.0.0)
     webmock (1.21.0)
       addressable (>= 2.3.6)
@@ -106,3 +106,6 @@ DEPENDENCIES
   rake (>= 10.3.2)
   test-unit (~> 3.0.2)
   webmock (>= 1.17.0)
+
+BUNDLED WITH
+   1.10.5

data/fluent-plugin-google-cloud.gemspec

@@ -4,7 +4,7 @@ Gem::Specification.new do |gem|
   gem.summary       = %q{Fluentd plugin to stream logs to the Google Cloud Platform's logging API}
   gem.homepage      = 'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache 2.0'
-  gem.version       = '0.4.1'
+  gem.version       = '0.4.2'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
 

data/lib/fluent/plugin/out_google_cloud.rb

@@ -54,6 +54,26 @@ module Fluent
     config_param :zone, :string, :default => nil
     config_param :vm_id, :string, :default => nil
 
+    # label_map (specified as a JSON object) is an unordered set of fluent
+    # field names whose values are sent as labels rather than as part of the
+    # struct payload.
+    #
+    # Each entry in the map is a {"field_name": "label_name"} pair.  When
+    # the "field_name" (as parsed by the input plugin) is encountered, a label
+    # with the corresponding "label_name" is added to the log entry.  The
+    # value of the field is used as the value of the label.
+    #
+    # The map gives the user additional flexibility in specifying label
+    # names, including the ability to use characters which would not be
+    # legal as part of fluent field names.
+    #
+    # Example:
+    #   label_map {
+    #     "field_name_1": "sent_label_name_1",
+    #     "field_name_2": "some.prefix/sent_label_name_2"
+    #   }
+    config_param :label_map, :hash, :default => nil
+
     # TODO: Add a log_name config option rather than just using the tag?
 
     # Expose attr_readers to make testing of metadata more direct than only
@@ -222,8 +242,9 @@ module Fluent
           'entries' => [],
         }
         arr.each do |time, record|
-          next unless record.is_a? Hash
+          next unless record.is_a?(Hash)
           if (record.has_key?('timestamp') &&
+              record['timestamp'].is_a?(Hash) &&
               record['timestamp'].has_key?('seconds') &&
               record['timestamp'].has_key?('nanos'))
             ts_secs = record['timestamp']['seconds']
@@ -270,6 +291,22 @@ module Fluent
             entry['metadata']['severity'] = 'DEFAULT'
           end
 
+          # If a field is present in the label_map, send its value as a label
+          # (mapping the field name to label name as specified in the config)
+          # and do not send that field as part of the payload.
+          if !label_map.nil?
+            labels = {}
+            @label_map.each do |field, label|
+              if record.has_key?(field)
+                labels[label] = record[field]
+                record.delete(field)
+              end
+            end
+            if !labels.empty?
+              entry['metadata']['labels'] = labels
+            end
+          end
+
           # use textPayload if the only remainaing key is 'message',
           # otherwise use a struct.
           if (record.size == 1 && record.has_key?('message'))
@@ -461,7 +498,7 @@ module Fluent
     def init_api_client
       @client = Google::APIClient.new(
         :application_name => 'Fluentd Google Cloud Logging plugin',
-        :application_version => '0.4.1',
+        :application_version => '0.4.2',
         :retries => 1)
 
       if @auth_method == 'private_key'

data/test/plugin/test_out_google_cloud.rb

@@ -165,7 +165,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
 
   def create_driver(conf=APPLICATION_DEFAULT_CONFIG)
     Fluent::Test::BufferedOutputTestDriver.new(
-        Fluent::GoogleCloudOutput).configure(conf)
+        Fluent::GoogleCloudOutput).configure(conf, use_v1_config: true)
   end
 
   def test_configure_service_account_application_default
@@ -459,6 +459,20 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     end
   end
 
+  def test_malformed_timestamp
+    setup_gce_metadata_stubs
+    setup_logging_stubs
+    d = create_driver()
+    # if timestamp is not a hash it is passed through to the struct payload.
+    d.emit({'message' => log_entry(0), 'timestamp' => 'not-a-hash'})
+    d.run
+    verify_index = 0
+    verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
+      assert_equal 2, entry['structPayload'].size, entry
+      assert_equal "not-a-hash", entry['structPayload']['timestamp'], entry
+    end
+  end
+
   def test_severities
     setup_gce_metadata_stubs
     setup_logging_stubs
@@ -481,6 +495,59 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     end
   end
 
+  def test_label_map_without_field_present
+    setup_gce_metadata_stubs
+    setup_logging_stubs
+    config = %[label_map { "label_field": "sent_label" }]
+    d = create_driver(config)
+    d.emit({'message' => log_entry(0)})
+    d.run
+    # No additional labels should be present
+    verify_log_entries(1, COMPUTE_PARAMS)
+  end
+
+  def test_label_map_with_field_present
+    setup_gce_metadata_stubs
+    setup_logging_stubs
+    config = %[label_map { "label_field": "sent_label" }]
+    d = create_driver(config)
+    d.emit({'message' => log_entry(0), 'label_field' => 'label_value'})
+    d.run
+    # make a deep copy of COMPUTE_PARAMS and add the parsed label.
+    params = Marshal.load(Marshal.dump(COMPUTE_PARAMS))
+    params['labels']['sent_label'] = 'label_value'
+    verify_log_entries(1, params)
+  end
+
+  def test_label_map_with_multiple_fields
+    setup_gce_metadata_stubs
+    setup_logging_stubs
+    config = %[
+      label_map {
+        "label1": "sent_label_1",
+        "label_number_two": "foo.googleapis.com/bar",
+        "label3": "label3"
+      }]
+    d = create_driver(config)
+    # not_a_label passes through to the struct payload
+    d.emit({'message' => log_entry(0),
+            'label1' => 'value1',
+            'label_number_two' => 'value2',
+            'not_a_label' => 'value4',
+            'label3' => 'value3'})
+    d.run
+    # make a deep copy of COMPUTE_PARAMS and add the parsed labels.
+    params = Marshal.load(Marshal.dump(COMPUTE_PARAMS))
+    params['labels']['sent_label_1'] = 'value1'
+    params['labels']['foo.googleapis.com/bar'] = 'value2'
+    params['labels']['label3'] = 'value3'
+    verify_log_entries(1, params, 'structPayload') do |entry|
+      assert_equal 2, entry['structPayload'].size, entry
+      assert_equal "test log entry 0", entry['structPayload']['message'], entry
+      assert_equal 'value4', entry['structPayload']['not_a_label'], entry
+    end
+  end
+
   def test_multiple_logs
     setup_gce_metadata_stubs
     setup_logging_stubs

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-11 00:00:00.000000000 Z
+date: 2015-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -194,3 +194,4 @@ test_files:
 - test/plugin/data/credentials.json
 - test/plugin/data/invalid_credentials.json
 - test/plugin/test_out_google_cloud.rb
+has_rdoc: