fluent-plugin-in-redis 0.0.1.pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. checksums.yaml +7 -0
  2. data/lib/fluent/plugin/in_redis.rb +135 -0
  3. metadata +153 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 568a729c9ead72546063b92cc92c69e659b0a9934df7fd5f248376080414a88f
4
+ data.tar.gz: 314411e6c5eba3b1b7b64b92588013d9457078a20f0927cc27c97dfc503e3ccc
5
+ SHA512:
6
+ metadata.gz: b1b2eb58b312e77a6f0c37e42ad485ea7016a26b870d03c2bdead4ea007924d5f1a7802a22bc7b6250e8f1e712beb8048f9a3ae8392270cce361b6e7e5b3a662
7
+ data.tar.gz: 659172ec716011b9704fb3303bf624afa08d1760b467891319e98a27e9bccefbd65b498ebb9b5800e9747b32fe600b226f25e07af419a16130c6a364010295ac
data/lib/fluent/plugin/in_redis.rb ADDED
@@ -0,0 +1,135 @@
1
+ # coding: utf-8
2
+ require 'fluent/input'
3
+ require 'redis'
4
+
5
+ class Fluent::RedisInInput < Fluent::Input
6
+ Fluent::Plugin.register_input('redis', self)
7
+
8
+ unless method_defined?(:log)
9
+ define_method('log') { $log }
10
+ end
11
+
12
+ unless method_defined?(:router)
13
+ define_method('router') { Fluent::Engine }
14
+ end
15
+
16
+ # Redis configurations.
17
+ config_param :url, :string # REQUIRED
18
+ # format --> redis://:[password]@[hostname]:[port]/[db]
19
+ config_param :db, :integer, default: 0
20
+ config_param :timeout, :float, default: 5.0
21
+
22
+ # TODO: Support other redis-rb configuration options as needed, see:
23
+ # https://www.rubydoc.info/github/redis/redis-rb/Redis:initialize
24
+
25
+ # Plugin configurations.
26
+ config_param :key, :string # REQUIRED
27
+ config_param :tag, :string, default: nil # If undefined, multi tag support will be enabled.
28
+ config_param :max_events, :integer, default: 100
29
+ config_param :poll_interval, :float, default: 5.0 # seconds
30
+
31
+ def initialize
32
+ super
33
+ end
34
+
35
+ # `configure` is called before `start`.
36
+ def configure(conf)
37
+ super
38
+
39
+ # Non-redis configurations
40
+ @key = conf["key"]
41
+ @max = conf["max_events"]
42
+ @int = conf["poll_interval"]
43
+ @tag = conf["tag"]
44
+
45
+ # redis configuration
46
+ redis_conf = {}
47
+ ["url", "timeout", "db" ].each do |key|
48
+ redis_conf[key.to_sym] = conf[key] if conf.has_key?(key)
49
+ end
50
+
51
+ @redis = Redis.new(redis_conf)
52
+ #raise Fluent::ConfigError, "failed to connect to redis source." unless @redis.connected?
53
+ end
54
+
55
+ # `start` is called when starting and after `configure` is successfully completed.
56
+ def start
57
+ super
58
+
59
+ # Async processor
60
+ @loop = Coolio::Loop.new
61
+
62
+ meth = @tag.nil? ? :multi_handler : :handler
63
+ timer = TimerWatcher.new(@int, true, log, &method(meth))
64
+
65
+ @loop.attach(timer)
66
+ @thread = Thread.new(&method(:run))
67
+ end
68
+
69
+ # `shutdown` is called while closing down.
70
+ def shutdown
71
+ #@running = false
72
+
73
+ @loop.watchers.each {|w| w.detach}
74
+ @loop.stop
75
+
76
+ super
77
+ end
78
+
79
+ protected
80
+ def run
81
+ @loop.run
82
+ rescue => e
83
+ log.error(e.message)
84
+ log.error_backtrace(e.backtrace)
85
+ end
86
+
87
+ # This is more efficent
88
+ def handler
89
+ records = @redis.rpop(@key, @max)
90
+
91
+ stream = Fluent::MultiEventStream.new
92
+ records.each do |record|
93
+ record.delete("@tag")
94
+ time = record.delete("@time")
95
+
96
+ stream.add(time, record)
97
+ end
98
+
99
+ router.emit_stream(@tag, stream)
100
+ end
101
+
102
+ # This is more flexible to real world use
103
+ def multi_handler
104
+ records = @redis.rpop(@key, @max)
105
+
106
+ streams = {}
107
+ records.each do |record|
108
+ tag = record.delete("@tag")
109
+ time = record.delete("@time")
110
+
111
+ streams[tag] ||= Fluent::MultiEventStream.new
112
+ streams[tag].add(time, record)
113
+ end
114
+
115
+ streams.each do |tag, stream|
116
+ router.emit_stream(tag, stream)
117
+ end
118
+ end
119
+
120
+ class TimerWatcher < Coolio::TimerWatcher
121
+ def initialize(interval, repeat, log, &callback)
122
+ @callback = callback
123
+ @log = log
124
+ super(interval, repeat)
125
+ end
126
+
127
+ def on_timer
128
+ @callback.call
129
+ rescue => e
130
+ @log.error(e.message)
131
+ @log.error_backtrace(e.backtrace)
132
+ end
133
+ end # TimerWatcher
134
+ end
135
+
metadata ADDED
@@ -0,0 +1,153 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-in-redis
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1.pre
5
+ platform: ruby
6
+ authors:
7
+ - Joshua Mervine
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2021-09-30 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.3'
20
+ - - ">="
21
+ - !ruby/object:Gem::Version
22
+ version: 1.3.2
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - "~>"
28
+ - !ruby/object:Gem::Version
29
+ version: '1.3'
30
+ - - ">="
31
+ - !ruby/object:Gem::Version
32
+ version: 1.3.2
33
+ - !ruby/object:Gem::Dependency
34
+ name: redis
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '4.4'
40
+ - - ">="
41
+ - !ruby/object:Gem::Version
42
+ version: 4.4.0
43
+ type: :runtime
44
+ prerelease: false
45
+ version_requirements: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - "~>"
48
+ - !ruby/object:Gem::Version
49
+ version: '4.4'
50
+ - - ">="
51
+ - !ruby/object:Gem::Version
52
+ version: 4.4.0
53
+ - !ruby/object:Gem::Dependency
54
+ name: bundler
55
+ requirement: !ruby/object:Gem::Requirement
56
+ requirements:
57
+ - - ">="
58
+ - !ruby/object:Gem::Version
59
+ version: '0'
60
+ type: :development
61
+ prerelease: false
62
+ version_requirements: !ruby/object:Gem::Requirement
63
+ requirements:
64
+ - - ">="
65
+ - !ruby/object:Gem::Version
66
+ version: '0'
67
+ - !ruby/object:Gem::Dependency
68
+ name: rake
69
+ requirement: !ruby/object:Gem::Requirement
70
+ requirements:
71
+ - - ">="
72
+ - !ruby/object:Gem::Version
73
+ version: '0'
74
+ type: :development
75
+ prerelease: false
76
+ version_requirements: !ruby/object:Gem::Requirement
77
+ requirements:
78
+ - - ">="
79
+ - !ruby/object:Gem::Version
80
+ version: '0'
81
+ - !ruby/object:Gem::Dependency
82
+ name: pry
83
+ requirement: !ruby/object:Gem::Requirement
84
+ requirements:
85
+ - - ">="
86
+ - !ruby/object:Gem::Version
87
+ version: '0'
88
+ type: :development
89
+ prerelease: false
90
+ version_requirements: !ruby/object:Gem::Requirement
91
+ requirements:
92
+ - - ">="
93
+ - !ruby/object:Gem::Version
94
+ version: '0'
95
+ - !ruby/object:Gem::Dependency
96
+ name: mock_redis
97
+ requirement: !ruby/object:Gem::Requirement
98
+ requirements:
99
+ - - ">="
100
+ - !ruby/object:Gem::Version
101
+ version: '0'
102
+ type: :development
103
+ prerelease: false
104
+ version_requirements: !ruby/object:Gem::Requirement
105
+ requirements:
106
+ - - ">="
107
+ - !ruby/object:Gem::Version
108
+ version: '0'
109
+ - !ruby/object:Gem::Dependency
110
+ name: test-unit
111
+ requirement: !ruby/object:Gem::Requirement
112
+ requirements:
113
+ - - "~>"
114
+ - !ruby/object:Gem::Version
115
+ version: '3'
116
+ type: :development
117
+ prerelease: false
118
+ version_requirements: !ruby/object:Gem::Requirement
119
+ requirements:
120
+ - - "~>"
121
+ - !ruby/object:Gem::Version
122
+ version: '3'
123
+ description:
124
+ email:
125
+ - jmervine@mulesoft.com
126
+ executables: []
127
+ extensions: []
128
+ extra_rdoc_files: []
129
+ files:
130
+ - lib/fluent/plugin/in_redis.rb
131
+ homepage:
132
+ licenses: []
133
+ metadata: {}
134
+ post_install_message:
135
+ rdoc_options: []
136
+ require_paths:
137
+ - lib
138
+ required_ruby_version: !ruby/object:Gem::Requirement
139
+ requirements:
140
+ - - ">="
141
+ - !ruby/object:Gem::Version
142
+ version: '0'
143
+ required_rubygems_version: !ruby/object:Gem::Requirement
144
+ requirements:
145
+ - - ">"
146
+ - !ruby/object:Gem::Version
147
+ version: 1.3.1
148
+ requirements: []
149
+ rubygems_version: 3.1.4
150
+ signing_key:
151
+ specification_version: 4
152
+ summary: Fluentd input plugin for reading events from redis.
153
+ test_files: []