fluent-plugin-redis-enrichment 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e523d3d718b67a60e86a6c37f7ba9a08d88ba6ee51570b03fcdb3600b24d9968
-  data.tar.gz: ee9e08c30219913043fff587734019a4896f64aaf26883cc9bdafa34d78b3f71
+  metadata.gz: 227357c5db343e088b546cdea0fd1bb84216ae5882f6d3a2a63c8518f7bf9f1a
+  data.tar.gz: 758d5b6a499e4e0a6c2f14bee5c5758869fe53da4629a1f5543d02e17ebd1f7e
 SHA512:
-  metadata.gz: 503448814a124ab6c3d912325887dfe87f9c0f1159ab1e34b821c968ab8305cdf9b32ec661a939083fc9b76f854143ac1899829741dbd61519a9879fde5c6f37
-  data.tar.gz: c141425d3b669a4464a10df035956cee8da5957aef5c8bb7d4b5a5478339fe644881859e4bc822f502aff6a522ea75fd9dece70798cf4fee3243b221dd2cd34b
+  metadata.gz: 741f97f7c880925e4908623ea957a1ae3118bf6722f916a32cdb52d6f04270634e0e68304a016b54bf7cebf6d3362fbed46f61978208031213a14f8800e5d73a
+  data.tar.gz: 55c7c0b111e0ef094a6037527fabfe7b8c09356a962ed932529034421de83e2f77dca27c3dae200967d3c26bbb2999093f11e4906a0c5fdb072487a61746dae4

data/fluent-plugin-redis-enrichment.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-redis-enrichment'
-  spec.version = '0.2.0'
+  spec.version = '0.3.0'
   spec.authors = ['Thomas Tych']
   spec.email   = ['thomas.tych@gmail.com']
 

data/lib/fluent/plugin/filter_redis_enrichment.rb

@@ -44,6 +44,7 @@ module Fluent
       DEFAULT_SENTINEL_PORT = 26_379
       DEFAULT_CACHE_TTL = 30 * 60
       DEFAULT_CACHE_SIZE = 5000
+      DEFAULT_CACHE_TYPE = :lazy
 
       desc 'Redis host'
       config_param :redis_host, :string, default: DEFAULT_REDIS_HOST
@@ -66,6 +67,8 @@ module Fluent
       desc 'Sentinel redis role'
       config_param :redis_role, :enum, list: %i[master slave replica], default: DEFAULT_REDIS_ROLE
 
+      desc 'local Cache type'
+      config_param :cache_type, :enum, list: %i[lazy full no], default: DEFAULT_CACHE_TYPE
       desc 'local Cache size'
       config_param :cache_size, :integer, default: DEFAULT_CACHE_SIZE
       desc 'local Cache ttl'
@@ -98,12 +101,13 @@ module Fluent
       def start
         super
 
-        @cache = Cache.new(**cache_options)
-        @redis = RedisPool.new(**redis_options)
+        @redis = RedisPool.new(log: log, **redis_options)
+        @cache = Cache.new(redis: @redis, log: log, **cache_options)
       end
 
       def shutdown
         @redis.quit
+        @cache.clean
 
         super
       end
@@ -114,7 +118,7 @@ module Fluent
                                                             time: time,
                                                             record: new_record })
         log.debug("filter_redis_enrichment: on tag:#{tag}, search #{expanded_key}")
-        redis = @cache.getset(expanded_key) { @redis.get(expanded_key) }
+        redis = @cache.get(expanded_key)
         new_record_record_enrichment = @placeholder_expander.expand(@record_enrichment,
                                                                     { tag: tag,
                                                                       time: time,
@@ -125,6 +129,7 @@ module Fluent
 
       def cache_options
         {
+          type: cache_type,
           size: cache_size,
           ttl: cache_ttl
         }
@@ -172,39 +177,102 @@ module Fluent
         value_str # emit as string
       end
 
-      class Cache
-        def initialize(size: DEFAULT_CACHE_SIZE, ttl: DEFAULT_CACHE_TTL)
-          @size = size
-          @ttl = ttl
+      module Cache
+        def self.new(redis:, type: DEFAULT_CACHE_TYPE, size: DEFAULT_CACHE_SIZE, ttl: DEFAULT_CACHE_TTL, log: nil)
+          klass = case type
+                  when :lazy then LazyCache
+                  when :full then FullCache
+                  else
+                    NoCache
+                  end
+          klass.new(redis: redis, size: size, ttl: ttl, log: log)
         end
 
-        def getset(key, &block)
-          cache.getset(key, &block)
+        class NoCache
+          attr_reader :log
+
+          def initialize(redis:, size: nil, ttl: nil, log: nil)
+            @redis = redis
+            @size = size
+            @ttl = ttl
+            @log = log
+          end
+
+          def get(key)
+            @redis.get(key)
+          end
+
+          def clean
+          end
         end
 
-        private
+        class LazyCache < NoCache
+          def initialize(redis:, size: DEFAULT_CACHE_SIZE, ttl: DEFAULT_CACHE_TTL, log: nil)
+            super
+          end
+
+          def get(key)
+            cache.getset(key) { @redis.get(key) }
+          end
 
-        def cache
-          @cache ||= if @size == 0 || @ttl == 0
-                       NoCache.new
-                     else
-                       LruRedux::TTL::ThreadSafeCache.new(@size, @ttl)
-                     end
+          def cache
+            @cache ||= LruRedux::TTL::ThreadSafeCache.new(@size, @ttl)
+          end
         end
-      end
 
-      class NoCache
-        def getset(_key)
-          yield
+        class FullCache < NoCache
+          def initialize(*args, **kwargs)
+            super
+
+            initialize_cache
+          end
+
+          def initialize_cache
+            @cache = {}
+            @cache_mutex = Mutex.new
+
+            reload
+            @timer = timer(@ttl, &method(:reload))
+          end
+
+          def reload
+            log.warn :RELOAD if log
+            new_cache_content = @redis.get_all
+            @cache_mutex.synchronize { @cache.replace(new_cache_content) }
+          end
+
+          def get(key)
+            @cache_mutex.synchronize { @cache[key] }
+          end
+
+          def timer(interval, &block)
+            Thread.new do
+              loop do
+                begin
+                  sleep interval
+                  block.call
+                rescue StandardError => e
+                  log.warn(e) if log
+                end
+              end
+            end
+          end
+
+          def clean
+            @timer.exit if @timer
+          end
         end
       end
 
       # proxy for Redis client
       #   allow extract caching of cache
       class RedisPool
+        attr_reader :log
+
         def initialize(sentinels: DEFAULT_SENTINELS, name: DEFAULT_SENTINEL_MASTER, role: DEFAULT_REDIS_ROLE,
                        host: DEFAULT_REDIS_HOST, port: DEFAULT_REDIS_PORT, db: DEFAULT_REDIS_DB,
-                       password: DEFAULT_REDIS_PASSWORD, timeout: DEFAULT_REDIS_TIMEOUT, pool_size: DEFAULT_REDIS_POOL)
+                       password: DEFAULT_REDIS_PASSWORD, timeout: DEFAULT_REDIS_TIMEOUT, pool_size: DEFAULT_REDIS_POOL,
+                       log: nil)
           @sentinels = sentinels
           @name = name
           @role = role
@@ -214,6 +282,7 @@ module Fluent
           @password = password
           @timeout = timeout
           @pool_size = pool_size
+          @log = log
         end
 
         def get(key)
@@ -225,11 +294,21 @@ module Fluent
           when 'string' then redis.get(key)
           when 'hash' then redis.hgetall(key)
           else
-            log.warn("redis key '#{key}' has an unmanaged type '#{key_type}'")
+            log.warn("redis key '#{key}' has an unmanaged type '#{key_type}'") if log
             nil
           end
         end
 
+        def get_all
+          redis.scan_each.with_object({}) do |key, all|
+            case @redis.type(key)
+            when 'hash' then all[key] = @redis.hgetall(key)
+            when 'string' then all[key] = @redis.get(key)
+            when 'set' then all[key] = @redis.smembers(key)
+            end
+          end
+        end
+
         def quit
           redis.quit
         end
@@ -296,9 +375,11 @@ module Fluent
         def expand(__str_to_eval__, tag: nil, time: nil, record: nil, redis: nil, **_extra)
           instance_eval(__str_to_eval__)
         rescue NoMethodError => e
+          log.warn("while expanding #{__str_to_eval__}: #{e}")
           nil
         rescue StandardError => e
           log.warn("while expanding #{__str_to_eval__}: #{e}")
+          nil
         end
 
         Object.instance_methods.each do |m|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-redis-enrichment
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Thomas Tych
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-11-23 00:00:00.000000000 Z
+date: 2023-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump
@@ -202,7 +202,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 5.0.5
-description: 
+description:
 email:
 - thomas.tych@gmail.com
 executables: []
@@ -221,7 +221,7 @@ licenses:
 - Apache-2.0
 metadata:
   rubygems_mfa_required: 'true'
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -236,8 +236,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
-signing_key: 
+rubygems_version: 3.4.6
+signing_key:
 specification_version: 4
 summary: fluentd plugin to do data enrichment with redis.
 test_files: []