fluent-plugin-google-cloud 0.5.2 → 0.5.3.grpc.alpha.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8109f72f6f16a0756b3a30c911e9343392051b96
-  data.tar.gz: e76c44a9c3e8cb63c6125f1b621e7985e3050918
+  metadata.gz: 4289702253d986d7711b6e0a97712b13702901c3
+  data.tar.gz: aac7a196ff97e4c70234772ad5755861d80549aa
 SHA512:
-  metadata.gz: aad0d625ddb42768a112e887bae953d601bd646d9003d7a7308a19a3e9e4afdbd61725bd56b844cefc7626a99dad067ea3ef6d68e44f06bc324ab7abe937bcd2
-  data.tar.gz: c880a81905b4ce2535f247a7bd1938e46082461f8c9f48a03716908219e6fe95234397a09ee4975ed21dc5c196b782d1efd1ebf499617c7fb989aa234efbf60e
+  metadata.gz: 4eb63be9e1eb8492ff627270ba13ab5b2d1cbe4dc1c01a0e2b7b14cc3192084d3d477ee9338c25c3af3ab8875e2203ee8e0d7fac2ddbda52fbbc6c5983959a16
+  data.tar.gz: 8fd692a696fec2163b7c1935cf68e488d3eb4d780ebe90bd5e22f53d4e06cab5d2dbc8dedca419a8218019ce13eacf9d15eebe76a71af40ad02514dec80887dc

data/Gemfile.lock

@@ -1,10 +1,12 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.5.2)
+    fluent-plugin-google-cloud (0.5.3.grpc.alpha.1)
       fluentd (~> 0.10, <= 0.13)
       google-api-client (> 0.9)
+      googleapis-common-protos (~> 1.1)
       googleauth (~> 0.4)
+      grpc (~> 0.15)
       json (~> 1.8)
 
 GEM
@@ -39,6 +41,9 @@ GEM
       representable (~> 2.3.0)
       retriable (~> 2.0)
       thor (~> 0.19)
+    google-protobuf (3.0.0.alpha.5.0.5.1)
+    googleapis-common-protos (1.1.1)
+      google-protobuf (~> 3.0.0.alpha.5.0)
     googleauth (0.5.1)
       faraday (~> 0.9)
       jwt (~> 1.4)
@@ -47,6 +52,9 @@ GEM
       multi_json (~> 1.11)
       os (~> 0.9)
       signet (~> 0.7)
+    grpc (0.15.0)
+      google-protobuf (~> 3.0.0.alpha.5.0.3)
+      googleauth (~> 0.5.1)
     hashdiff (0.3.0)
     http_parser.rb (0.6.0)
     httpclient (2.8.0)

data/Rakefile

@@ -18,15 +18,25 @@ end
 
 # Building the gem will use the local file mode, so ensure it's world-readable.
 # https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud/issues/53
-desc 'Check plugin file permissions'
-task :check_perms do
-  plugin = 'lib/fluent/plugin/out_google_cloud.rb'
-  mode = File.stat(plugin).mode & 0777
-  fail "Unexpected mode #{mode.to_s(8)} for #{plugin}" unless
-    mode & 0444 == 0444
+desc 'Fix file permissions'
+task :fix_perms do
+  files = [
+    'lib/fluent/plugin/out_google_cloud.rb',
+    'lib/google/**/*.rb'
+  ].flat_map do |file|
+    file.include?('*') ? Dir.glob(file) : [file]
+  end
+
+  files.each do |file|
+    mode = File.stat(file).mode & 0777
+    next unless mode & 0444 != 0444
+    puts "Changing mode of #{file} from #{mode.to_s(8)} to "\
+         "#{(mode | 0444).to_s(8)}"
+    chmod mode | 0444, file
+  end
 end
 
 desc 'Run unit tests and RuboCop to check for style violations'
-task all: [:test, :rubocop, :check_perms]
+task all: [:test, :rubocop, :fix_perms]
 
 task default: :all

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      = \
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.5.2'
+  gem.version       = '0.5.3.grpc.alpha.1'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
@@ -20,8 +20,10 @@ eos
   gem.require_paths = ['lib']
 
   gem.add_runtime_dependency 'fluentd', '~> 0.10', '<= 0.13'
+  gem.add_runtime_dependency 'googleapis-common-protos', '~> 1.1'
   gem.add_runtime_dependency 'google-api-client', '> 0.9'
   gem.add_runtime_dependency 'googleauth', '~> 0.4'
+  gem.add_runtime_dependency 'grpc', '~> 0.15'
   gem.add_runtime_dependency 'json', '~> 1.8'
 
   gem.add_development_dependency 'mocha', '~> 1.1'

data/lib/fluent/plugin/out_google_cloud.rb

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+require 'grpc'
 require 'json'
 require 'open-uri'
 require 'socket'
@@ -18,6 +19,9 @@ require 'time'
 require 'yaml'
 require 'google/apis'
 require 'google/apis/logging_v1beta3'
+require 'google/logging/v1/logging'
+require 'google/logging/v1/logging_services'
+require 'google/logging/v1/log_entry'
 require 'googleauth'
 
 module Fluent
@@ -26,7 +30,7 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'
-    PLUGIN_VERSION = '0.5.2'
+    PLUGIN_VERSION = '0.5.3.grpc.alpha.1'
 
     # Constants for service names.
     APPENGINE_SERVICE = 'appengine.googleapis.com'
@@ -114,6 +118,10 @@ module Fluent
     #   }
     config_param :labels, :hash, :default => nil
 
+    # Whether to use gRPC instead of REST/JSON to communicate to the
+    # Cloud Logging API.
+    config_param :use_grpc, :bool, :default => false
+
     # DEPRECATED: The following parameters, if present in the config
     # indicate that the plugin configuration must be updated.
     config_param :auth_method, :string, :default => nil
@@ -348,23 +356,34 @@ module Fluent
             end
           end
         end
+
         arr.each do |time, record|
           next unless record.is_a?(Hash)
 
-          entry = Google::Apis::LoggingV1beta3::LogEntry.new(
-            metadata: Google::Apis::LoggingV1beta3::LogEntryMetadata.new(
-              service_name: @service_name,
-              project_id: @project_id,
-              zone: @zone,
-              labels: {}
-            ))
+          if @use_grpc
+            entry = Google::Logging::V1::LogEntry.new(
+              metadata: Google::Logging::V1::LogEntryMetadata.new(
+                service_name: @service_name.encode('utf-8'),
+                project_id: @project_id.encode('utf-8'),
+                zone: @zone.encode('utf-8'),
+                labels: {}
+              ))
+          else
+            entry = Google::Apis::LoggingV1beta3::LogEntry.new(
+              metadata: Google::Apis::LoggingV1beta3::LogEntryMetadata.new(
+                service_name: @service_name,
+                project_id: @project_id,
+                zone: @zone,
+                labels: {}
+              ))
+          end
 
           if @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
             @cloudfunctions_log_match =
               @cloudfunctions_log_regexp.match(record['log'])
           end
           if @service_name == CONTAINER_SERVICE
-            # Move the stdout/stderr annotation from the record into a label
+            # Move the stdout/stderr annotation from the record into a label.
             field_to_label(record, 'stream', entry.metadata.labels,
                            "#{CONTAINER_SERVICE}/stream")
             # If the record has been annotated by the kubernetes_metadata_filter
@@ -397,9 +416,28 @@ module Fluent
             end
           end
 
-          set_timestamp(record, entry, time)
-          set_severity(record, entry)
-          set_http_request(record, entry)
+          ts_secs, ts_nanos = compute_timestamp(record, time)
+          if @use_grpc
+            entry.metadata.timestamp = Google::Protobuf::Timestamp.new(
+              seconds: ts_secs,
+              nanos: ts_nanos
+            )
+
+            entry.metadata.severity =
+              grpc_severity(compute_severity(record, entry))
+
+            set_http_request_grpc(record, entry) # FIXME
+          else
+            entry.metadata.timestamp = {
+              seconds: ts_secs,
+              nanos: ts_nanos
+            }
+
+            entry.metadata.severity =
+              compute_severity(record, entry)
+
+            set_http_request(record, entry)
+          end
 
           # If a field is present in the label_map, send its value as a label
           # (mapping the field name to label name as specified in the config)
@@ -417,8 +455,12 @@ module Fluent
               @cloudfunctions_log_match['execution_id']
           end
 
-          set_payload(record, entry, is_json)
-          entry.metadata.labels = nil if entry.metadata.labels.empty?
+          if @use_grpc
+            set_payload_grpc(record, entry, is_json)
+          else
+            set_payload(record, entry, is_json)
+            entry.metadata.labels = nil if entry.metadata.labels.empty?
+          end
 
           entries.push(entry)
         end
@@ -427,52 +469,449 @@ module Fluent
 
         log_name = log_name(tag, labels)
 
-        begin
-          # Does the actual write to the cloud logging api.
-          # The URI of the write is constructed by the Google::Api request;
-          # it is equivalent to this URL:
-          # 'https://logging.googleapis.com/v1beta3/projects/' \
-          #   "#{@project_id}/logs/#{log_name}/entries:write"
-
-          client = api_client
-
-          write_request = \
-            Google::Apis::LoggingV1beta3::WriteLogEntriesRequest.new(
-              common_labels: labels,
-              entries: entries)
-
-          # TODO: RequestOptions
-          client.write_log_entries(@project_id, log_name, write_request)
-
-          # Let the user explicitly know when the first call succeeded,
-          # to aid with verification and troubleshooting.
-          unless @successful_call
-            @successful_call = true
-            @log.info 'Successfully sent to Google Cloud Logging API.'
+        if @use_grpc
+          begin
+            # Does the actual write to the cloud logging api.
+
+            client = api_client
+
+            labels_utf8_pairs = labels.map do |k, v|
+              [k.encode('utf-8'), v.encode('utf-8')]
+            end
+            utf8_log_name = log_name.encode('utf-8')
+
+            write_request = Google::Logging::V1::WriteLogEntriesRequest.new(
+              log_name: "projects/#{@project_id}/logs/#{utf8_log_name}",
+              common_labels: Hash[labels_utf8_pairs],
+              entries: entries
+            )
+
+            client.write_log_entries(write_request)
+
+            # Let the user explicitly know when the first call succeeded,
+            # to aid with verification and troubleshooting.
+            unless @successful_call
+              @successful_call = true
+              @log.info 'Successfully sent gRPC to Google Cloud Logging API.'
+            end
+
+          rescue GRPC::Cancelled => error
+            # RPC cancelled, so retry via re-raising the error.
+            raise error
+
+          rescue GRPC::BadStatus => error
+            case error.code
+            when GRPC::Core::StatusCodes::CANCELLED,
+                 GRPC::Core::StatusCodes::UNAVAILABLE,
+                 GRPC::Core::StatusCodes::DEADLINE_EXCEEDED,
+                 GRPC::Core::StatusCodes::INTERNAL,
+                 GRPC::Core::StatusCodes::UNKNOWN
+              # TODO
+              # Server error, so retry via re-raising the error.
+              raise error
+            when GRPC::Core::StatusCodes::UNIMPLEMENTED,
+                 GRPC::Core::StatusCodes::RESOURCE_EXHAUSTED
+              # Most client errors indicate a problem with the request itself
+              # and should not be retried.
+              dropped = entries.length
+              @log.warn "Dropping #{dropped} log message(s)",
+                        error: error.to_s, error_code: error.code.to_s
+            when GRPC::Core::StatusCodes::UNAUTHENTICATED
+              # Authorization error.
+              # These are usually solved via a `gcloud auth` call, or by
+              # modifying the permissions on the Google Cloud project.
+              dropped = entries.length
+              @log.warn "Dropping #{dropped} log message(s)",
+                        error: error.to_s, error_code: error.code.to_s
+            else
+              # Assume this is a problem with the request itself
+              # and don't retry.
+              dropped = entries.length
+              @log.error "Unknown response code #{error.code} from the "\
+                         "server, dropping #{dropped} log message(s)",
+                         error: error.to_s, error_code: error.code.to_s
+            end
           end
+        else
+          begin
+            # Does the actual write to the cloud logging api.
+
+            client = api_client
+
+            # The URI of the write is constructed by the Google::Api request;
+            # it is equivalent to this URL:
+            # 'https://logging.googleapis.com/v1beta3/projects/' \
+            #   "#{@project_id}/logs/#{log_name}/entries:write"
+            write_request = \
+              Google::Apis::LoggingV1beta3::WriteLogEntriesRequest.new(
+                common_labels: labels,
+                entries: entries)
+
+            # TODO: RequestOptions
+            client.write_log_entries(@project_id, log_name, write_request)
+
+            # Let the user explicitly know when the first call succeeded,
+            # to aid with verification and troubleshooting.
+            unless @successful_call
+              @successful_call = true
+              @log.info 'Successfully sent to Google Cloud Logging API.'
+            end
 
-        rescue Google::Apis::ServerError => error
-          # Server error, so retry via re-raising the error.
-          raise error
-
-        rescue Google::Apis::AuthorizationError => error
-          # Authorization error.
-          # These are usually solved via a `gcloud auth` call, or by modifying
-          # the permissions on the Google Cloud project.
-          dropped = entries.length
-          @log.warn "Dropping #{dropped} log message(s)",
-                    error_class: error.class.to_s, error: error.to_s
-
-        rescue Google::Apis::ClientError => error
-          # Most ClientErrors indicate a problem with the request itself and
-          # should not be retried.
-          dropped = entries.length
-          @log.warn "Dropping #{dropped} log message(s)",
-                    error_class: error.class.to_s, error: error.to_s
+          rescue Google::Apis::ServerError => error
+            # Server error, so retry via re-raising the error.
+            raise error
+
+          rescue Google::Apis::AuthorizationError => error
+            # Authorization error.
+            # These are usually solved via a `gcloud auth` call, or by modifying
+            # the permissions on the Google Cloud project.
+            dropped = entries.length
+            @log.warn "Dropping #{dropped} log message(s)",
+                      error_class: error.class.to_s, error: error.to_s
+
+          rescue Google::Apis::ClientError => error
+            # Most ClientErrors indicate a problem with the request itself and
+            # should not be retried.
+            dropped = entries.length
+            @log.warn "Dropping #{dropped} log message(s)",
+                      error_class: error.class.to_s, error: error.to_s
+          end
         end
       end
     end
 
+    # def old_write(chunk)
+    #   # Group the entries since we have to make one call per tag.
+    #   grouped_entries = {}
+    #   chunk.msgpack_each do |tag, *arr|
+    #     grouped_entries[tag] = [] unless grouped_entries.key?(tag)
+    #     grouped_entries[tag].push(arr)
+    #   end
+    #
+    #   grouped_entries.each do |tag, arr|
+    #     entries = []
+    #     labels = @common_labels.clone
+    #
+    #     if @running_cloudfunctions
+    #       # If the current group of entries is coming from a Cloud Functions
+    #       # function, the function name can be extracted from the tag.
+    #       match_data = @cloudfunctions_tag_regexp.match(tag)
+    #       if match_data
+    #         # Service name is set to Cloud Functions only for logs actually
+    #         # coming from a function.
+    #         @service_name = CLOUDFUNCTIONS_SERVICE
+    #         labels["#{CLOUDFUNCTIONS_SERVICE}/region"] = @gcf_region
+    #         labels["#{CLOUDFUNCTIONS_SERVICE}/function_name"] =
+    #           decode_cloudfunctions_function_name(
+    #             match_data['encoded_function_name'])
+    #       else
+    #         # Other logs are considered as coming from the Container Engine
+    #         # service.
+    #         @service_name = CONTAINER_SERVICE
+    #       end
+    #     end
+    #     if @service_name == CONTAINER_SERVICE && \
+    #        @compiled_kubernetes_tag_regexp
+    #       # Container logs in Kubernetes are tagged based on where they came
+    #       # from, so we can extract useful metadata from the tag.
+    #       # Do this here to avoid having to repeat it for each record.
+    #       match_data = @compiled_kubernetes_tag_regexp.match(tag)
+    #       if match_data
+    #         %w(namespace_name pod_name container_name).each do |field|
+    #           labels["#{CONTAINER_SERVICE}/#{field}"] = match_data[field]
+    #         end
+    #       end
+    #     end
+    #
+    #     if @use_grpc
+    #       arr.each do |time, record|
+    #         next unless record.is_a?(Hash)
+    #
+    #         entry = Google::Logging::V1::LogEntry.new(
+    #           metadata: Google::Logging::V1::LogEntryMetadata.new(
+    #             service_name: @service_name.encode('utf-8'),
+    #             project_id: @project_id.encode('utf-8'),
+    #             zone: @zone.encode('utf-8'),
+    #             labels: {}
+    #           ))
+    #
+    #         if @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
+    #           @cloudfunctions_log_match =
+    #             @cloudfunctions_log_regexp.match(record['log'])
+    #         end
+    #         if @service_name == CONTAINER_SERVICE
+    #           # Move the stdout/stderr annotation from the record into a
+    #           # label.
+    #           field_to_label(record, 'stream', entry.metadata.labels,
+    #                          "#{CONTAINER_SERVICE}/stream")
+    #           # If the record has been annotated by the
+    #           # kubernetes_metadata_filter
+    #           # plugin, then use that metadata.  Otherwise, rely on
+    #           # commonLabels
+    #           # populated at the grouped_entries level from the group's tag.
+    #           if record.key?('kubernetes')
+    #             handle_container_metadata(record, entry)
+    #           end
+    #
+    #           # Save the timestamp if available, then clear it out to allow
+    #           # for determining whether we should parse the log or message
+    #           # field.
+    #           timestamp = record.key?('time') ? record['time'] : nil
+    #           record.delete('time')
+    #           # If the log is json, we want to export it as a structured log
+    #           # unless there is additional metadata that would be lost.
+    #           is_json = false
+    #           if record.length == 1 && record.key?('log')
+    #             record_json = parse_json_or_nil(record['log'])
+    #           end
+    #           if record.length == 1 && record.key?('message')
+    #             record_json = parse_json_or_nil(record['message'])
+    #           end
+    #           unless record_json.nil?
+    #             record = record_json
+    #             is_json = true
+    #           end
+    #           # Restore timestamp if necessary.
+    #           unless record.key?('time') || timestamp.nil?
+    #             record['time'] = timestamp
+    #           end
+    #         end
+    #
+    #         ts_secs, ts_nanos = compute_timestamp(record, time)
+    #         entry.metadata.timestamp = Google::Protobuf::Timestamp.new(
+    #           seconds: ts_secs,
+    #           nanos: ts_nanos
+    #         )
+    #
+    #         entry.metadata.severity =
+    #           grpc_severity(compute_severity(record, entry))
+    #
+    #         set_http_request_grpc(record, entry) # FIXME
+    #
+    #         # If a field is present in the label_map, send its value as a
+    #         # label
+    #         # (mapping the field name to label name as specified in the
+    #         # config)
+    #         # and do not send that field as part of the payload.
+    #         if @label_map
+    #           @label_map.each do |field, label|
+    #             field_to_label(record, field, entry.metadata.labels, label)
+    #           end
+    #         end
+    #
+    #         if @service_name == CLOUDFUNCTIONS_SERVICE &&
+    #            @cloudfunctions_log_match &&
+    #            @cloudfunctions_log_match['execution_id']
+    #           entry.metadata.labels['execution_id'] =
+    #             @cloudfunctions_log_match['execution_id']
+    #         end
+    #
+    #         set_payload_grpc(record, entry, is_json)
+    #
+    #         entries.push(entry)
+    #       end
+    #       # Don't send an empty request if we rejected all the entries.
+    #       next if entries.empty?
+    #
+    #       log_name = log_name(tag, labels)
+    #
+    #       begin
+    #         # Does the actual write to the cloud logging api.
+    #
+    #         client = api_client
+    #
+    #         labels_utf8_pairs = labels.map do |k, v|
+    #           [k.encode('utf-8'), v.encode('utf-8')]
+    #         end
+    #
+    #         write_request = Google::Logging::V1::WriteLogEntriesRequest.new(
+    #           log_name: log_name.encode('utf-8'),
+    #           common_labels: Hash[labels_utf8_pairs],
+    #           entries: entries
+    #         )
+    #
+    #         client.write_log_entries(write_request)
+    #
+    #         # Let the user explicitly know when the first call succeeded,
+    #         # to aid with verification and troubleshooting.
+    #         unless @successful_call
+    #           @successful_call = true
+    #           @log.info 'Successfully sent gRPC to Google Cloud Logging API.'
+    #         end
+    #
+    #       rescue GRPC::Cancelled => error
+    #         # RPC cancelled, so retry via re-raising the error.
+    #         raise error
+    #
+    #       rescue GRPC::BadStatus => error
+    #         case error.code
+    #         when GRPC::Core::StatusCodes::CANCELLED,
+    #              GRPC::Core::StatusCodes::UNAVAILABLE,
+    #              GRPC::Core::StatusCodes::DEADLINE_EXCEEDED,
+    #              GRPC::Core::StatusCodes::INTERNAL,
+    #              GRPC::Core::StatusCodes::UNKNOWN
+    #           # TODO
+    #           # Server error, so retry via re-raising the error.
+    #           raise error
+    #         when GRPC::Core::StatusCodes::UNIMPLEMENTED,
+    #              GRPC::Core::StatusCodes::RESOURCE_EXHAUSTED
+    #           # Most client errors indicate a problem with the request itself
+    #           # and should not be retried.
+    #           dropped = entries.length
+    #           @log.warn "Dropping #{dropped} log message(s)",
+    #                     error: error.to_s, error_code: error.code.to_s
+    #         when GRPC::Core::StatusCodes::UNAUTHENTICATED
+    #           # Authorization error.
+    #           # These are usually solved via a `gcloud auth` call, or by
+    #           # modifying the permissions on the Google Cloud project.
+    #           dropped = entries.length
+    #           @log.warn "Dropping #{dropped} log message(s)",
+    #                     error: error.to_s, error_code: error.code.to_s
+    #         else
+    #           @log.error "Unknown response code #{error.code} from the " \
+    #                      "server",
+    #                      error: error.to_s, error_code: error.code.to_s
+    #         end
+    #       end
+    #     else
+    #       arr.each do |time, record|
+    #         next unless record.is_a?(Hash)
+    #
+    #         entry = Google::Apis::LoggingV1beta3::LogEntry.new(
+    #           metadata: Google::Apis::LoggingV1beta3::LogEntryMetadata.new(
+    #             service_name: @service_name,
+    #             project_id: @project_id,
+    #             zone: @zone,
+    #             labels: {}
+    #           ))
+    #
+    #         if @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
+    #           @cloudfunctions_log_match =
+    #             @cloudfunctions_log_regexp.match(record['log'])
+    #         end
+    #         if @service_name == CONTAINER_SERVICE
+    #           # Move the stdout/stderr annotation from the record into a label
+    #           field_to_label(record, 'stream', entry.metadata.labels,
+    #                          "#{CONTAINER_SERVICE}/stream")
+    #           # If the record has been annotated by the
+    #           # kubernetes_metadata_filter
+    #           # plugin, then use that metadata. Otherwise, rely on
+    #           # commonLabels
+    #           # populated at the grouped_entries level from the group's tag.
+    #           if record.key?('kubernetes')
+    #             handle_container_metadata(record, entry)
+    #           end
+    #
+    #           # Save the timestamp if available, then clear it out to allow
+    #           # for determining whether we should parse the log or message
+    #           # field.
+    #           timestamp = record.key?('time') ? record['time'] : nil
+    #           record.delete('time')
+    #           # If the log is json, we want to export it as a structured log
+    #           # unless there is additional metadata that would be lost.
+    #           is_json = false
+    #           if record.length == 1 && record.key?('log')
+    #             record_json = parse_json_or_nil(record['log'])
+    #           end
+    #           if record.length == 1 && record.key?('message')
+    #             record_json = parse_json_or_nil(record['message'])
+    #           end
+    #           unless record_json.nil?
+    #             record = record_json
+    #             is_json = true
+    #           end
+    #           # Restore timestamp if necessary.
+    #           unless record.key?('time') || timestamp.nil?
+    #             record['time'] = timestamp
+    #           end
+    #         end
+    #
+    #         ts_secs, ts_nanos = compute_timestamp(record, time)
+    #         entry.metadata.timestamp = {
+    #           seconds: ts_secs,
+    #           nanos: ts_nanos
+    #         }
+    #
+    #         entry.metadata.severity = compute_severity(record, entry)
+    #
+    #         set_http_request(record, entry)
+    #
+    #         # If a field is present in the label_map, send its value as a
+    #         # label
+    #         # (mapping the field name to label name as specified in the
+    #         # config)
+    #         # and do not send that field as part of the payload.
+    #         if @label_map
+    #           @label_map.each do |field, label|
+    #             field_to_label(record, field, entry.metadata.labels, label)
+    #           end
+    #         end
+    #
+    #         if @service_name == CLOUDFUNCTIONS_SERVICE &&
+    #            @cloudfunctions_log_match &&
+    #            @cloudfunctions_log_match['execution_id']
+    #           entry.metadata.labels['execution_id'] =
+    #             @cloudfunctions_log_match['execution_id']
+    #         end
+    #
+    #         set_payload(record, entry, is_json)
+    #         entry.metadata.labels = nil if entry.metadata.labels.empty?
+    #
+    #         entries.push(entry)
+    #       end
+    #       # Don't send an empty request if we rejected all the entries.
+    #       next if entries.empty?
+    #
+    #       log_name = log_name(tag, labels)
+    #
+    #       begin
+    #         # Does the actual write to the cloud logging api.
+    #         # The URI of the write is constructed by the Google::Api request;
+    #         # it is equivalent to this URL:
+    #         # 'https://logging.googleapis.com/v1beta3/projects/' \
+    #         #   "#{@project_id}/logs/#{log_name}/entries:write"
+    #
+    #         client = api_client
+    #
+    #         write_request = \
+    #           Google::Apis::LoggingV1beta3::WriteLogEntriesRequest.new(
+    #             common_labels: labels,
+    #             entries: entries)
+    #
+    #         # TODO: RequestOptions
+    #         client.write_log_entries(@project_id, log_name, write_request)
+    #
+    #         # Let the user explicitly know when the first call succeeded,
+    #         # to aid with verification and troubleshooting.
+    #         unless @successful_call
+    #           @successful_call = true
+    #           @log.info 'Successfully sent to Google Cloud Logging API.'
+    #         end
+    #
+    #       rescue Google::Apis::ServerError => error
+    #         # Server error, so retry via re-raising the error.
+    #         raise error
+    #
+    #       rescue Google::Apis::AuthorizationError => error
+    #         # Authorization error.
+    #         # These are usually solved via a `gcloud auth` call, or by
+    #         # modifying
+    #         # the permissions on the Google Cloud project.
+    #         dropped = entries.length
+    #         @log.warn "Dropping #{dropped} log message(s)",
+    #                   error_class: error.class.to_s, error: error.to_s
+    #
+    #       rescue Google::Apis::ClientError => error
+    #         # Most ClientErrors indicate a problem with the request itself and
+    #         # should not be retried.
+    #         dropped = entries.length
+    #         @log.warn "Dropping #{dropped} log message(s)",
+    #                   error_class: error.class.to_s, error: error.to_s
+    #       end
+    #     end
+    #   end
+    # end
+
     private
 
     def parse_json_or_nil(input)
@@ -602,7 +1041,7 @@ module Fluent
       instance_prefix
     end
 
-    def set_timestamp(record, entry, time)
+    def compute_timestamp(record, time)
       if record.key?('timestamp') &&
          record['timestamp'].is_a?(Hash) &&
          record['timestamp'].key?('seconds') &&
@@ -645,41 +1084,36 @@ module Fluent
         ts_secs = timestamp.tv_sec
         ts_nanos = timestamp.tv_nsec
       end
-      entry.metadata.timestamp = {
-        seconds: ts_secs,
-        nanos: ts_nanos
-      }
+      [ts_secs, ts_nanos]
     end
 
-    def set_severity(record, entry)
+    def compute_severity(record, entry)
       if @service_name == CLOUDFUNCTIONS_SERVICE
         if @cloudfunctions_log_match && @cloudfunctions_log_match['severity']
-          entry.metadata.severity =
-            parse_severity(@cloudfunctions_log_match['severity'])
+          return parse_severity(@cloudfunctions_log_match['severity'])
         elsif record.key?('stream') && record['stream'] == 'stdout'
-          entry.metadata.severity = 'INFO'
           record.delete('stream')
+          return 'INFO'
         elsif record.key?('stream') && record['stream'] == 'stderr'
-          entry.metadata.severity = 'ERROR'
           record.delete('stream')
+          return 'ERROR'
         else
-          entry.metadata.severity = 'DEFAULT'
+          return 'DEFAULT'
         end
       elsif record.key?('severity')
-        entry.metadata.severity = parse_severity(record['severity'])
-        record.delete('severity')
+        return parse_severity(record.delete('severity'))
       elsif @service_name == CONTAINER_SERVICE && \
             entry.metadata.labels.key?("#{CONTAINER_SERVICE}/stream")
         stream = entry.metadata.labels["#{CONTAINER_SERVICE}/stream"]
         if stream == 'stdout'
-          entry.metadata.severity = 'INFO'
+          return 'INFO'
         elsif stream == 'stderr'
-          entry.metadata.severity = 'ERROR'
+          return 'ERROR'
         else
-          entry.metadata.severity = 'DEFAULT'
+          return 'DEFAULT'
         end
       else
-        entry.metadata.severity = 'DEFAULT'
+        return 'DEFAULT'
       end
     end
 
@@ -702,6 +1136,25 @@ module Fluent
       entry.http_request = output
     end
 
+    def set_http_request_grpc(record, entry)
+      return nil unless record['httpRequest'].is_a?(Hash)
+      input = record['httpRequest']
+      output = Google::Logging::Type::HttpRequest.new
+      output.request_method = input.delete('requestMethod')
+      output.request_url = input.delete('requestUrl')
+      output.request_size = input.delete('requestSize').to_i
+      output.status = input.delete('status').to_i
+      output.response_size = input.delete('responseSize').to_i
+      output.user_agent = input.delete('userAgent')
+      output.remote_ip = input.delete('remoteIp')
+      output.referer = input.delete('referer')
+      output.cache_hit = input.delete('cacheHit') == 'true'
+      output.validated_with_origin_server = \
+        input.delete('validatedWithOriginServer') == 'true'
+      record.delete('httpRequest') if input.empty?
+      entry.http_request = output
+    end
+
     # Values permitted by the API for 'severity' (which is an enum).
     VALID_SEVERITIES = Set.new(
       %w(DEFAULT DEBUG INFO NOTICE WARNING ERROR CRITICAL ALERT EMERGENCY))
@@ -762,6 +1215,38 @@ module Fluent
       'DEFAULT'
     end
 
+    GRPC_SEVERITY_MAPPING = {
+      'DEFAULT' => Google::Logging::Type::LogSeverity::DEFAULT,
+      'DEBUG' => Google::Logging::Type::LogSeverity::DEBUG,
+      'INFO' => Google::Logging::Type::LogSeverity::INFO,
+      'NOTICE' => Google::Logging::Type::LogSeverity::NOTICE,
+      'WARNING' => Google::Logging::Type::LogSeverity::WARNING,
+      'ERROR' => Google::Logging::Type::LogSeverity::ERROR,
+      'CRITICAL' => Google::Logging::Type::LogSeverity::CRITICAL,
+      'ALERT' => Google::Logging::Type::LogSeverity::ALERT,
+      'EMERGENCY' => Google::Logging::Type::LogSeverity::EMERGENCY,
+      0 => Google::Logging::Type::LogSeverity::DEFAULT,
+      100 => Google::Logging::Type::LogSeverity::DEBUG,
+      200 => Google::Logging::Type::LogSeverity::INFO,
+      300 => Google::Logging::Type::LogSeverity::NOTICE,
+      400 => Google::Logging::Type::LogSeverity::WARNING,
+      500 => Google::Logging::Type::LogSeverity::ERROR,
+      600 => Google::Logging::Type::LogSeverity::CRITICAL,
+      700 => Google::Logging::Type::LogSeverity::ALERT,
+      800 => Google::Logging::Type::LogSeverity::EMERGENCY
+    }
+
+    def grpc_severity(severity)
+      # TODO: find out why this doesn't work.
+      # if severity.is_a? String
+      #   return Google::Logging::Type::LogSeverity.resolve(severity)
+      # end
+      if GRPC_SEVERITY_MAPPING.key?(severity)
+        return GRPC_SEVERITY_MAPPING[severity]
+      end
+      severity
+    end
+
     def decode_cloudfunctions_function_name(function_name)
       function_name.gsub(/c\.[a-z]/) { |s| s.upcase[-1] }
         .gsub('u.u', '_').gsub('d.d', '$').gsub('a.a', '@').gsub('p.p', '.')
@@ -815,6 +1300,72 @@ module Fluent
       end
     end
 
+    def value_from_ruby(value)
+      ret = Google::Protobuf::Value.new
+      case value
+      when NilClass
+        ret.null_value = 0
+      when Numeric
+        ret.number_value = value
+      when String
+        ret.string_value = value.encode('utf-8')
+      when TrueClass
+        ret.bool_value = true
+      when FalseClass
+        ret.bool_value = false
+      when Google::Protobuf::Struct
+        ret.struct_value = value
+      when Hash
+        ret.struct_value = struct_from_ruby(value)
+      when Google::Protobuf::ListValue
+        ret.list_value = value
+      when Array
+        ret.list_value = list_from_ruby(value)
+      else
+        @log.error "Unknown type: #{value.class}"
+        fail Google::Protobuf::Error, "Unknown type: #{value.class}"
+      end
+      ret
+    end
+
+    def list_from_ruby(arr)
+      ret = Google::Protobuf::ListValue.new
+      arr.each do |v|
+        ret.values << value_from_ruby(v)
+      end
+      ret
+    end
+
+    def struct_from_ruby(hash)
+      ret = Google::Protobuf::Struct.new
+      hash.each do |k, v|
+        ret.fields[k] ||= value_from_ruby(v)
+      end
+      ret
+    end
+
+    def set_payload_grpc(record, entry, is_json)
+      # If this is a Cloud Functions log that matched the expected regexp,
+      # use text payload. Otherwise, use JSON if we found valid JSON, or text
+      # payload in the following cases:
+      # 1. This is a Cloud Functions log and the 'log' key is available
+      # 2. This is an unstructured Container log and the 'log' key is available
+      # 3. The only remaining key is 'message'
+      if @service_name == CLOUDFUNCTIONS_SERVICE && @cloudfunctions_log_match
+        entry.text_payload = @cloudfunctions_log_match['text']
+      elsif @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
+        entry.text_payload = record['log']
+      elsif is_json
+        entry.struct_payload = struct_from_ruby(record)
+      elsif @service_name == CONTAINER_SERVICE && record.key?('log')
+        entry.text_payload = record['log']
+      elsif record.size == 1 && record.key?('message')
+        entry.text_payload = record['message']
+      else
+        entry.struct_payload = struct_from_ruby(record)
+      end
+    end
+
     def log_name(tag, common_labels)
       if @service_name == CLOUDFUNCTIONS_SERVICE
         return 'cloud-functions'
@@ -833,6 +1384,7 @@ module Fluent
     end
 
     def init_api_client
+      return if @use_grpc
       # TODO: Use a non-default ClientOptions object.
       Google::Apis::ClientOptions.default.application_name = PLUGIN_NAME
       Google::Apis::ClientOptions.default.application_version = PLUGIN_VERSION
@@ -842,14 +1394,23 @@ module Fluent
     end
 
     def api_client
-      unless @client.authorization.expired?
-        begin
-          @client.authorization.fetch_access_token!
-        rescue MultiJson::ParseError
-          # Workaround an issue in the API client; just re-raise a more
-          # descriptive error for the user (which will still cause a retry).
-          raise Google::APIClient::ClientError, 'Unable to fetch access ' \
-            'token (no scopes configured?)'
+      if @use_grpc
+        ssl_creds = GRPC::Core::ChannelCredentials.new
+        authentication = Google::Auth.get_application_default
+        creds = GRPC::Core::CallCredentials.new(authentication.updater_proc)
+        creds = ssl_creds.compose(creds)
+        @client = Google::Logging::V1::LoggingService::Stub.new(
+          'logging.googleapis.com', creds)
+      else
+        unless @client.authorization.expired?
+          begin
+            @client.authorization.fetch_access_token!
+          rescue MultiJson::ParseError
+            # Workaround an issue in the API client; just re-raise a more
+            # descriptive error for the user (which will still cause a retry).
+            raise Google::APIClient::ClientError, 'Unable to fetch access ' \
+              'token (no scopes configured?)'
+          end
         end
       end
       @client