fluent-plugin-sidekiq 0.0.5 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/VERSION +1 -1
- data/lib/fluent/plugin/out_sidekiq.rb +86 -83
- metadata +21 -22
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 8f504b306f8ff6c58c7d4301481958095054d689b44a0eb47814baa496bbbc15
|
4
|
+
data.tar.gz: 1d3622dbb05fca9ca6cc58dc55e3005acd90f8be778a698c00feda974bb0c54a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d266e1e43afa813a59f827439b58fc2b32a6a40e125060b76db612d32d735defce90cc33e41f410a826e2dc66af67f7bb27ab5e4523300a4164977bf46408fb4
|
7
|
+
data.tar.gz: df123481dddee992b042d27ecd75fc40c3581ed695d1a22375259741a9b9f76d9bad7c179213af1e9a094cf93a44b7389fc57302fe9833b0043f747940d3cab0
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0
|
1
|
+
0.1.0
|
@@ -1,102 +1,105 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
class Fluent::SidekiqOutput < Fluent::BufferedOutput
|
4
|
-
class Batch
|
5
|
-
attr_accessor :queue, :klass, :payload
|
1
|
+
# frozen_string_literal: true
|
6
2
|
|
7
|
-
|
8
|
-
|
9
|
-
self.klass = k
|
10
|
-
self.payload = {:class => k, :retry => true, :jid => "", :enqueued_at => 0, :args => [[]] }
|
11
|
-
end
|
3
|
+
require 'fluent/plugin/output'
|
4
|
+
require "redis"
|
12
5
|
|
13
|
-
|
14
|
-
|
15
|
-
|
6
|
+
module Fluent
|
7
|
+
module Plugin
|
8
|
+
class SidekiqOutput < Output
|
9
|
+
class Batch
|
10
|
+
attr_accessor :queue, :klass, :payload
|
16
11
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
end
|
12
|
+
def initialize(q, k)
|
13
|
+
self.queue = q
|
14
|
+
self.klass = k
|
15
|
+
self.payload = {:class => k, :retry => true, :jid => "", :enqueued_at => 0, :args => [[]] }
|
16
|
+
end
|
23
17
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
end
|
28
|
-
end
|
18
|
+
def acceptable_batch(q, c, p, max_size)
|
19
|
+
q == queue && c == klass && (payload[:args][0].length + p.length) <= max_size
|
20
|
+
end
|
29
21
|
|
30
|
-
|
31
|
-
|
22
|
+
def add_to_batch(p)
|
23
|
+
payload[:retry] = p['retry']
|
24
|
+
payload[:jid] = p['jid']
|
25
|
+
payload[:enqueued_at] = p['enqueued_at']
|
26
|
+
payload[:args][0] += p['args'][0]
|
27
|
+
end
|
32
28
|
|
33
|
-
|
34
|
-
|
35
|
-
|
29
|
+
def enqueue(client)
|
30
|
+
client.sadd("queues", queue)
|
31
|
+
client.lpush("queue:#{queue}", JSON.generate(payload))
|
32
|
+
end
|
33
|
+
end
|
36
34
|
|
37
|
-
|
38
|
-
|
39
|
-
end
|
35
|
+
VERSION = "0.1.0"
|
36
|
+
Fluent::Plugin.register_output("sidekiq", self)
|
40
37
|
|
41
|
-
|
42
|
-
|
43
|
-
|
38
|
+
config_param :redis_url, :string, :default => 'redis://localhost:6379'
|
39
|
+
config_param :redis_namespace, :string, :default => nil
|
40
|
+
config_param :max_batch_size, :integer, :default => 100
|
44
41
|
|
45
|
-
|
46
|
-
|
47
|
-
|
42
|
+
def format(tag, time, record)
|
43
|
+
[tag, time, record].to_msgpack
|
44
|
+
end
|
48
45
|
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
if @redis_namespace
|
53
|
-
require "redis/namespace"
|
54
|
-
Redis::Namespace.new(@redis_namespace, client)
|
55
|
-
else
|
56
|
-
client
|
57
|
-
end
|
58
|
-
end
|
46
|
+
def formatted_to_msgpack_binary
|
47
|
+
true
|
48
|
+
end
|
59
49
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
else
|
71
|
-
payload = JSON.parse(data['payload'])
|
72
|
-
if payload['args'][0].kind_of?(Array) && payload['args'][0].all? { |a| a.kind_of?(Hash) }
|
73
|
-
queue = data.delete('queue')
|
74
|
-
klass = payload['class']
|
50
|
+
def redis_client
|
51
|
+
opts = {url: @redis_url, driver: :hiredis}
|
52
|
+
client = Redis.new(opts)
|
53
|
+
if @redis_namespace
|
54
|
+
require "redis/namespace"
|
55
|
+
Redis::Namespace.new(@redis_namespace, client)
|
56
|
+
else
|
57
|
+
client
|
58
|
+
end
|
59
|
+
end
|
75
60
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
61
|
+
def write(chunk)
|
62
|
+
client = redis_client
|
63
|
+
scheduled_jobs = []
|
64
|
+
batches = []
|
65
|
+
manually_queue = []
|
66
|
+
|
67
|
+
chunk.msgpack_each do |tag, time, data|
|
68
|
+
at = data.delete('at')
|
69
|
+
if at
|
70
|
+
scheduled_jobs << [at, data['payload']]
|
71
|
+
else
|
72
|
+
payload = JSON.parse(data['payload'])
|
73
|
+
if payload['args'][0].kind_of?(Array) && payload['args'][0].all? { |a| a.kind_of?(Hash) }
|
74
|
+
queue = data.delete('queue')
|
75
|
+
klass = payload['class']
|
76
|
+
|
77
|
+
batch = batches.find { |b| b.acceptable_batch(queue, klass, payload['args'][0], max_batch_size) }
|
78
|
+
if !batch
|
79
|
+
batch = Batch.new(queue, klass)
|
80
|
+
batches << batch
|
81
|
+
end
|
82
|
+
|
83
|
+
batch.add_to_batch(payload)
|
84
|
+
else
|
85
|
+
# Manual queuing
|
86
|
+
manually_queue << data
|
87
|
+
end
|
80
88
|
end
|
89
|
+
end
|
81
90
|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
91
|
+
if scheduled_jobs.length > 0
|
92
|
+
client.zadd('schedule', scheduled_jobs)
|
93
|
+
end
|
94
|
+
batches.each do |batch|
|
95
|
+
batch.enqueue(client)
|
96
|
+
end
|
97
|
+
manually_queue.each do |data|
|
98
|
+
queue = data.delete('queue')
|
99
|
+
client.sadd('queues', queue)
|
100
|
+
client.lpush("queue:#{queue}", data['payload'])
|
86
101
|
end
|
87
102
|
end
|
88
103
|
end
|
89
|
-
|
90
|
-
if scheduled_jobs.length > 0
|
91
|
-
client.zadd('schedule', scheduled_jobs)
|
92
|
-
end
|
93
|
-
batches.each do |batch|
|
94
|
-
batch.enqueue(client)
|
95
|
-
end
|
96
|
-
manually_queue.each do |data|
|
97
|
-
queue = data.delete('queue')
|
98
|
-
client.sadd('queues', queue)
|
99
|
-
client.lpush("queue:#{queue}", data['payload'])
|
100
|
-
end
|
101
104
|
end
|
102
105
|
end
|
metadata
CHANGED
@@ -1,83 +1,83 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-sidekiq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alex Scarborough
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-08-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - ~>
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '3.0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - ~>
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '3.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: hiredis
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: bundler
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - ~>
|
45
|
+
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '1.3'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - ~>
|
52
|
+
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '1.3'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: rake
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- -
|
59
|
+
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '0'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- -
|
66
|
+
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: fluentd
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- -
|
73
|
+
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '0'
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- -
|
80
|
+
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
83
|
description: Sidekiq plugin for Fluentd
|
@@ -87,9 +87,9 @@ executables: []
|
|
87
87
|
extensions: []
|
88
88
|
extra_rdoc_files: []
|
89
89
|
files:
|
90
|
-
- .gitignore
|
91
|
-
- .ruby-gemset
|
92
|
-
- .ruby-version
|
90
|
+
- ".gitignore"
|
91
|
+
- ".ruby-gemset"
|
92
|
+
- ".ruby-version"
|
93
93
|
- Gemfile
|
94
94
|
- LICENSE.txt
|
95
95
|
- README.md
|
@@ -101,24 +101,23 @@ homepage: https://github.com/GoCarrot/fluent-plugin-sidekiq
|
|
101
101
|
licenses:
|
102
102
|
- MIT
|
103
103
|
metadata: {}
|
104
|
-
post_install_message:
|
104
|
+
post_install_message:
|
105
105
|
rdoc_options: []
|
106
106
|
require_paths:
|
107
107
|
- lib
|
108
108
|
required_ruby_version: !ruby/object:Gem::Requirement
|
109
109
|
requirements:
|
110
|
-
- -
|
110
|
+
- - ">="
|
111
111
|
- !ruby/object:Gem::Version
|
112
112
|
version: '0'
|
113
113
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
114
114
|
requirements:
|
115
|
-
- -
|
115
|
+
- - ">="
|
116
116
|
- !ruby/object:Gem::Version
|
117
117
|
version: '0'
|
118
118
|
requirements: []
|
119
|
-
|
120
|
-
|
121
|
-
signing_key:
|
119
|
+
rubygems_version: 3.2.3
|
120
|
+
signing_key:
|
122
121
|
specification_version: 4
|
123
122
|
summary: Sidekiq plugin for Fluentd
|
124
123
|
test_files: []
|