fluent-plugin-envoy-parser 0.0.6

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7c10f3da0b761807d3252acd763508bec26b911e
+  data.tar.gz: 34240bda20b56c637067307607d96003a4f976ff
+SHA512:
+  metadata.gz: 7614a862d3d707776c069cc1ad73db8b13c509c45ae3da508e63a89cbfa200ac21cfc58e476e5d73b94b62fabacd1919fb493395db074aa47b382674b819a3d6
+  data.tar.gz: f83af30b11ba26488adcdd2bd8394769c5d001c842ed0ae2b4b6887b2f854820cc945391af92ec1c5cd62d1b61155c38854ad8eac95109ae390349e61839ea15

data/lib/fluent/plugin/parser_envoy.rb

@@ -0,0 +1,116 @@
+require 'fluent/plugin/parser'
+
+module Fluent
+  module Plugin
+    class EnvoyParser < Parser
+      Plugin.register_parser('envoy', self)
+
+      config_param :log_format, :string, :default => 'envoy_http'
+      config_param :time_key, :string, :default => 'time'
+
+      @REGEXP = nil
+      DEFAULT_REGEXP = /^\[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\.)*?)(?: +\S*)?) (?<protocol>\S+)?" (?<response_code>\S+) (?<response_flags>\S+) (?<bytes_received>\S+) (?<bytes_sent>\S+) (?<duration>\S+) (?<x_envoy_upstream_service_time>\S+) "(?<x_forwarded_for>[^\"]*)" "(?<user_agent>[^\"]*)" "(?<x_request_id>[^\"]*)" "(?<authority>[^\"]*)" "(?<upstream_host>[^\"]*)"?$/
+
+      TIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
+
+      def initialize
+        super
+        @mutex = Mutex.new
+      end
+
+      def configure(conf={})
+        super
+        @REGEXP = /$^/ 
+
+        if @log_format == 'envoy_http' or @log_format == 'envoy_tcp'
+          @REGEXP = DEFAULT_REGEXP
+        end
+
+        @time_parser = time_parser_create(format: TIME_FORMAT)
+      end
+
+      def patterns
+        {'format' => @REGEXP, 'time_format' => TIME_FORMAT}
+      end
+
+      def parse(text)
+        m = @REGEXP.match(text)
+        unless m
+          yield nil, nil
+          return
+        end
+
+        time = m['time']
+        time = @mutex.synchronize { @time_parser.parse(time) }
+
+        if @log_format == 'envoy_http'
+            method = m['method']
+            path = m['path']
+            protocol = m['protocol']
+            response_flags = m['response_flags']
+
+            response_code = m['response_code'].to_i
+            response_code = nil if response_code == 0
+
+            bytes_sent = m['bytes_sent']
+            bytes_sent = (bytes_sent == '-') ? nil : bytes_sent.to_i
+
+            bytes_received = m['bytes_received']
+            bytes_received = (bytes_received == '-') ? nil : bytes_received.to_i
+
+            duration = m['duration']
+            duration = (duration == '-') ? nil : (Float(duration)/1000).to_s + "s"
+
+            x_envoy_upstream_service_time = m['x_envoy_upstream_service_time']
+            x_request_id = m['x_request_id']
+            authority = m['authority']
+            upstream_host = m['upstream_host']
+
+            user_agent = m['user_agent']
+            user_agent = (user_agent == '-') ? nil : user_agent
+
+            x_forwarded_for = m['x_forwarded_for']
+            x_forwarded_for = (x_forwarded_for == '-') ? nil : x_forwarded_for
+
+            record = {
+              "method" => method,
+              "path" => path,
+              "protocol" => protocol,
+              "response_code" => response_code,
+              "response_flags" => response_flags,
+              "bytes_received" => bytes_received,
+              "bytes_sent" => bytes_sent,
+              "duration" => duration,
+              "x_envoy_upstream_service_time" => x_envoy_upstream_service_time,
+              "x_forwarded_for" => x_forwarded_for,
+              "user_agent" => user_agent,
+              "authority" => authority,
+              "upstream_host" => upstream_host
+            }
+          elsif @log_format == 'envoy_tcp'
+
+            bytes_sent = m['bytes_sent']
+            bytes_sent = (bytes_sent == '-') ? nil : bytes_sent.to_i
+
+            bytes_received = m['bytes_received']
+            bytes_received = (bytes_received == '-') ? nil : bytes_received.to_i
+
+            duration = m['duration']
+            duration = (duration == '-') ? nil : (Float(duration)/1000).to_s + "s"
+
+            upstream_host = m['upstream_host']
+
+            record = {
+              "bytes_received" => bytes_received,
+              "bytes_sent" => bytes_sent,
+              "duration" => duration,
+              "upstream_host" => upstream_host
+            }
+        end
+        record["time"] = m['time'] if @keep_time_key
+
+        yield time, record
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-envoy-parser
+version: !ruby/object:Gem::Version
+  version: 0.0.6
+platform: ruby
+authors:
+- salrashid123
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-01-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.15.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.15.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 11.1.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 11.1.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+description: Fluentd parser plugin to parse standard Envoy Proxy access logs
+email:
+- salrashid123@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/fluent/plugin/parser_envoy.rb
+homepage: https://github.com/salrashid123/fluent-plugin-envoy-parser
+licenses:
+- Apache License, Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2.2
+signing_key: 
+specification_version: 4
+summary: Fluentd parser plugin to parse envoy HTTP/TCP access logs for Fluentd and
+  Google Cloud Logging
+test_files: []