fluent-plugin-in-redis 0.0.1.pre

Sign up to get free protection for your applications and to get access to all the features.
Files changed (3) hide show
  1. checksums.yaml +7 -0
  2. data/lib/fluent/plugin/in_redis.rb +135 -0
  3. metadata +153 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 568a729c9ead72546063b92cc92c69e659b0a9934df7fd5f248376080414a88f
4
+ data.tar.gz: 314411e6c5eba3b1b7b64b92588013d9457078a20f0927cc27c97dfc503e3ccc
5
+ SHA512:
6
+ metadata.gz: b1b2eb58b312e77a6f0c37e42ad485ea7016a26b870d03c2bdead4ea007924d5f1a7802a22bc7b6250e8f1e712beb8048f9a3ae8392270cce361b6e7e5b3a662
7
+ data.tar.gz: 659172ec716011b9704fb3303bf624afa08d1760b467891319e98a27e9bccefbd65b498ebb9b5800e9747b32fe600b226f25e07af419a16130c6a364010295ac
data/lib/fluent/plugin/in_redis.rb ADDED
@@ -0,0 +1,135 @@
1
+ # coding: utf-8
2
+ require 'fluent/input'
3
+ require 'redis'
4
+
5
+ class Fluent::RedisInInput < Fluent::Input
6
+ Fluent::Plugin.register_input('redis', self)
7
+
8
+ unless method_defined?(:log)
9
+ define_method('log') { $log }
10
+ end
11
+
12
+ unless method_defined?(:router)
13
+ define_method('router') { Fluent::Engine }
14
+ end
15
+
16
+ # Redis configurations.
17
+ config_param :url, :string # REQUIRED
18
+ # format --> redis://:[password]@[hostname]:[port]/[db]
19
+ config_param :db, :integer, default: 0
20
+ config_param :timeout, :float, default: 5.0
21
+
22
+ # TODO: Support other redis-rb configuration options as needed, see:
23
+ # https://www.rubydoc.info/github/redis/redis-rb/Redis:initialize
24
+
25
+ # Plugin configurations.
26
+ config_param :key, :string # REQUIRED
27
+ config_param :tag, :string, default: nil # If undefined, multi tag support will be enabled.
28
+ config_param :max_events, :integer, default: 100
29
+ config_param :poll_interval, :float, default: 5.0 # seconds
30
+
31
+ def initialize
32
+ super
33
+ end
34
+
35
+ # `configure` is called before `start`.
36
+ def configure(conf)
37
+ super
38
+
39
+ # Non-redis configurations
40
+ @key = conf["key"]
41
+ @max = conf["max_events"]
42
+ @int = conf["poll_interval"]
43
+ @tag = conf["tag"]
44
+
45
+ # redis configuration
46
+ redis_conf = {}
47
+ ["url", "timeout", "db" ].each do |key|
48
+ redis_conf[key.to_sym] = conf[key] if conf.has_key?(key)
49
+ end
50
+
51
+ @redis = Redis.new(redis_conf)
52
+ #raise Fluent::ConfigError, "failed to connect to redis source." unless @redis.connected?
53
+ end
54
+
55
+ # `start` is called when starting and after `configure` is successfully completed.
56
+ def start
57
+ super
58
+
59
+ # Async processor
60
+ @loop = Coolio::Loop.new
61
+
62
+ meth = @tag.nil? ? :multi_handler : :handler
63
+ timer = TimerWatcher.new(@int, true, log, &method(meth))
64
+
65
+ @loop.attach(timer)
66
+ @thread = Thread.new(&method(:run))
67
+ end
68
+
69
+ # `shutdown` is called while closing down.
70
+ def shutdown
71
+ #@running = false
72
+
73
+ @loop.watchers.each {|w| w.detach}
74
+ @loop.stop
75
+
76
+ super
77
+ end
78
+
79
+ protected
80
+ def run
81
+ @loop.run
82
+ rescue => e
83
+ log.error(e.message)
84
+ log.error_backtrace(e.backtrace)
85
+ end
86
+
87
+ # This is more efficent
88
+ def handler
89
+ records = @redis.rpop(@key, @max)
90
+
91
+ stream = Fluent::MultiEventStream.new
92
+ records.each do |record|
93
+ record.delete("@tag")
94
+ time = record.delete("@time")
95
+
96
+ stream.add(time, record)
97
+ end
98
+
99
+ router.emit_stream(@tag, stream)
100
+ end
101
+
102
+ # This is more flexible to real world use
103
+ def multi_handler
104
+ records = @redis.rpop(@key, @max)
105
+
106
+ streams = {}
107
+ records.each do |record|
108
+ tag = record.delete("@tag")
109
+ time = record.delete("@time")
110
+
111
+ streams[tag] ||= Fluent::MultiEventStream.new
112
+ streams[tag].add(time, record)
113
+ end
114
+
115
+ streams.each do |tag, stream|
116
+ router.emit_stream(tag, stream)
117
+ end
118
+ end
119
+
120
+ class TimerWatcher < Coolio::TimerWatcher
121
+ def initialize(interval, repeat, log, &callback)
122
+ @callback = callback
123
+ @log = log
124
+ super(interval, repeat)
125
+ end
126
+
127
+ def on_timer
128
+ @callback.call
129
+ rescue => e
130
+ @log.error(e.message)
131
+ @log.error_backtrace(e.backtrace)
132
+ end
133
+ end # TimerWatcher
134
+ end
135
+
metadata ADDED
@@ -0,0 +1,153 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-in-redis
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1.pre
5
+ platform: ruby
6
+ authors:
7
+ - Joshua Mervine
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2021-09-30 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.3'
20
+ - - ">="
21
+ - !ruby/object:Gem::Version
22
+ version: 1.3.2
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - "~>"
28
+ - !ruby/object:Gem::Version
29
+ version: '1.3'
30
+ - - ">="
31
+ - !ruby/object:Gem::Version
32
+ version: 1.3.2
33
+ - !ruby/object:Gem::Dependency
34
+ name: redis
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '4.4'
40
+ - - ">="
41
+ - !ruby/object:Gem::Version
42
+ version: 4.4.0
43
+ type: :runtime
44
+ prerelease: false
45
+ version_requirements: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - "~>"
48
+ - !ruby/object:Gem::Version
49
+ version: '4.4'
50
+ - - ">="
51
+ - !ruby/object:Gem::Version
52
+ version: 4.4.0
53
+ - !ruby/object:Gem::Dependency
54
+ name: bundler
55
+ requirement: !ruby/object:Gem::Requirement
56
+ requirements:
57
+ - - ">="
58
+ - !ruby/object:Gem::Version
59
+ version: '0'
60
+ type: :development
61
+ prerelease: false
62
+ version_requirements: !ruby/object:Gem::Requirement
63
+ requirements:
64
+ - - ">="
65
+ - !ruby/object:Gem::Version
66
+ version: '0'
67
+ - !ruby/object:Gem::Dependency
68
+ name: rake
69
+ requirement: !ruby/object:Gem::Requirement
70
+ requirements:
71
+ - - ">="
72
+ - !ruby/object:Gem::Version
73
+ version: '0'
74
+ type: :development
75
+ prerelease: false
76
+ version_requirements: !ruby/object:Gem::Requirement
77
+ requirements:
78
+ - - ">="
79
+ - !ruby/object:Gem::Version
80
+ version: '0'
81
+ - !ruby/object:Gem::Dependency
82
+ name: pry
83
+ requirement: !ruby/object:Gem::Requirement
84
+ requirements:
85
+ - - ">="
86
+ - !ruby/object:Gem::Version
87
+ version: '0'
88
+ type: :development
89
+ prerelease: false
90
+ version_requirements: !ruby/object:Gem::Requirement
91
+ requirements:
92
+ - - ">="
93
+ - !ruby/object:Gem::Version
94
+ version: '0'
95
+ - !ruby/object:Gem::Dependency
96
+ name: mock_redis
97
+ requirement: !ruby/object:Gem::Requirement
98
+ requirements:
99
+ - - ">="
100
+ - !ruby/object:Gem::Version
101
+ version: '0'
102
+ type: :development
103
+ prerelease: false
104
+ version_requirements: !ruby/object:Gem::Requirement
105
+ requirements:
106
+ - - ">="
107
+ - !ruby/object:Gem::Version
108
+ version: '0'
109
+ - !ruby/object:Gem::Dependency
110
+ name: test-unit
111
+ requirement: !ruby/object:Gem::Requirement
112
+ requirements:
113
+ - - "~>"
114
+ - !ruby/object:Gem::Version
115
+ version: '3'
116
+ type: :development
117
+ prerelease: false
118
+ version_requirements: !ruby/object:Gem::Requirement
119
+ requirements:
120
+ - - "~>"
121
+ - !ruby/object:Gem::Version
122
+ version: '3'
123
+ description:
124
+ email:
125
+ - jmervine@mulesoft.com
126
+ executables: []
127
+ extensions: []
128
+ extra_rdoc_files: []
129
+ files:
130
+ - lib/fluent/plugin/in_redis.rb
131
+ homepage:
132
+ licenses: []
133
+ metadata: {}
134
+ post_install_message:
135
+ rdoc_options: []
136
+ require_paths:
137
+ - lib
138
+ required_ruby_version: !ruby/object:Gem::Requirement
139
+ requirements:
140
+ - - ">="
141
+ - !ruby/object:Gem::Version
142
+ version: '0'
143
+ required_rubygems_version: !ruby/object:Gem::Requirement
144
+ requirements:
145
+ - - ">"
146
+ - !ruby/object:Gem::Version
147
+ version: 1.3.1
148
+ requirements: []
149
+ rubygems_version: 3.1.4
150
+ signing_key:
151
+ specification_version: 4
152
+ summary: Fluentd input plugin for reading events from redis.
153
+ test_files: []