fluent-plugin-druid-log 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: efa3398798dfb2d5a258360dc86f8f76cd40fff9c3e454f6d60aa6ec2a60ffc4
-  data.tar.gz: 9b65a364cd585eef36c33e7090db7f34d3cf4df57fb6333d8cb09d9930dfbcd0
+  metadata.gz: d468b5e88c20ec40b45bac3d2e68eff09882d885df618fc57a3f7025f6613972
+  data.tar.gz: d53dfa9917eda7f46741f34afef2d16f4a9127df484eaf2406fbbd0272efec7d
 SHA512:
-  metadata.gz: 8a684926b58b86cb8f26c0d14825bb3bddd6392502796214dab3901e540f5677caeff85592bbd1a5c01e35d8f67de050e451bc23200615ae0a7e681d66c157a0
-  data.tar.gz: b02b37823c3699a85c3dd6bf7867f40ddd09a4b477d11194b2590e3cf44fd2295be1082b8bb8e0a25324a109f70f6df35adc5c6ace6d8f7f421aa57cb7d0e744
+  metadata.gz: cfe4e4749ee7f6e1767f4eb249cf6d9855c5a1703a498c34df7a2c4c1c2fe693ec88576a1402ea4fa9587f01f3f61954702f4e2ddc4a2d916f6fed2a7b3ee38c
+  data.tar.gz: 42b43f812473cbff5ec96c4f10b12ec36740526aeed4e80d67a059917c6e4056c05a623788db5164ef5c18e4e32e11f6d4bad9239003942e4d241d72af03f6e6

data/README.md

@@ -20,7 +20,7 @@ Example:
 
   <parse>
     @type regexp
-	expression /^(?<timestamp>[^\t]+)\t(?<remote_addr>[^\t]*)\t{1,2}(?<query_result>[^\t]+)\t(?<raw_query>.*)$/
+	expression /^(?<timestamp>[^\t]+)\t(?<remote_addr>[^\t]*)\t{1,2}(?<query_result>[^\t]+)\t(?<query>.*)$/
 	time_key timestamp
 	keep_time_key true
   </parse>

data/lib/fluent/plugin/filter_format_druid_audit_log_1.rb

@@ -11,9 +11,12 @@ module Fluent
       helpers :event_emitter, :timer
 
       DEFAULT_QUERY_KEY = 'query'
+      DEFAULT_QUERY_RESULT_KEY = 'query_result'
 
       desc 'Query key'
       config_param :query_key, :string, default: DEFAULT_QUERY_KEY
+      desc 'Query result key'
+      config_param :query_result_key, :string, default: DEFAULT_QUERY_RESULT_KEY
 
       def configure(conf)
         super
@@ -28,18 +31,22 @@ module Fluent
       end
 
       def filter(_tag, _time, record)
-        new_record = format_record(record)
+        new_record = format_record(record.dup)
         fix_record(new_record)
         new_record
       end
 
       def format_record(record)
+        [query_key, query_result_key].each do |key|
+          record[key] = JSON.parse(record[key]) if record[key].is_a? String
+        end
+
         query_type = guess_query_type(record)
+        record['query_type'] = query_type
 
-        new_record = record.except(query_key)
-        new_record['query_type'] = query_type
-        new_record["#{query_type}_query".downcase] = record[query_key].dup
-        new_record
+        query_data = record.delete(query_key)
+        record["#{query_type}_query".downcase] = query_data
+        record
       end
 
       def guess_query_type(record)

data/lib/fluent/plugin/filter_format_druid_audit_log_2.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'fluent/plugin/filter'
+
+module Fluent
+  module Plugin
+    class FormatDruidAuditLog2Filter < Fluent::Plugin::Filter
+      NAME = 'format_druid_audit_log_2'
+      Fluent::Plugin.register_filter(NAME, self)
+
+      helpers :event_emitter, :timer
+
+      DEFAULT_QUERY_KEY = 'query'
+      DEFAULT_QUERY_RESULT_KEY = 'query_result'
+
+      desc 'Query key'
+      config_param :query_key, :string, default: DEFAULT_QUERY_KEY
+      desc 'Query result key'
+      config_param :query_result_key, :string, default: DEFAULT_QUERY_RESULT_KEY
+
+      def configure(conf)
+        super
+
+        return unless query_key.nil? and query_result.nil?
+
+        raise Fluent::ConfigError, 'query_key should be specified'
+      end
+
+      def multi_workers_ready?
+        true
+      end
+
+      def filter(_tag, _time, record)
+        new_record = format_record(record.dup)
+        fix_record(new_record)
+        new_record
+      end
+
+      def format_record(record)
+        [query_key, query_result_key].each do |key|
+          record[key] = JSON.parse(record[key]) if record[key].is_a? String
+        end
+
+        query_type = guess_query_type(record)
+        record['query_type'] = query_type
+
+        record
+      end
+
+      def guess_query_type(record)
+        record.dig(query_key,
+                   'queryType') || (record.dig('query_result',
+                                               'sqlQuery/time') && 'sql') || (record.dig(query_key,
+                                                                                         'query') && 'sql') || 'unknown'
+      end
+
+      def fix_record(record)
+        fix_record_query_granularity(record)
+      end
+
+      def fix_record_query_granularity(record)
+        return if record.dig(query_key, 'granularity').nil?
+
+        record[query_key]['granularity'] = record[query_key]['granularity'].to_s
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-druid-log
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Thomas Tych
@@ -211,6 +211,7 @@ files:
 - README.md
 - Rakefile
 - lib/fluent/plugin/filter_format_druid_audit_log_1.rb
+- lib/fluent/plugin/filter_format_druid_audit_log_2.rb
 homepage: https://gitlab.com/ttych/fluent-plugin-druid-log
 licenses:
 - Apache-2.0