fluent-plugin-record-demux 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.rubocop.yml +7 -5
  4. data/README.md +17 -11
  5. data/fluent-plugin-record-demux.gemspec +13 -13
  6. data/lib/fluent/plugin/out_record_demux.rb +13 -13
  7. metadata +52 -33
  8. data/Gemfile.lock +0 -131
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c137a1ee4d131ce2bce46dbef0093e9505db2692b11b2f996a12261a0e6b3acd
4
- data.tar.gz: 4f249f03b8b5aa1f2c2d735bab7cf5583d651f4065a22b636236977a8d1c61fb
3
+ metadata.gz: e3af2e3acf245f6e85007a6832dec1cdeb7fb7e54fc91afc2312e5c9da4be9be
4
+ data.tar.gz: 8296a752f7a84b20f900a214abc55bea52af25b4e9a06d1d079b445921331e39
5
5
  SHA512:
6
- metadata.gz: 27e72866fcba75e5710f12dba49e414b0e878cbe5df25031328c868dd2c51fdbb69c15650b7c8d8cb8570687cdfa5ffd051d9cb6fd47ebf97430d64289c126b1
7
- data.tar.gz: 4b30cf0a8befb3beea9ee1c92572e0f52ce4dcbd6f500bed9657cae383dafc17775b9c102039297a82e5a413e7d22ac1c89abfb38c6700b20ef602f437eadc62
6
+ metadata.gz: 2d99ca520a2a8e281769c3dcc508937e9ec34e4c925e54c848590aa4f59ed4ed0a773730847b697ef3e3d511d2437077531f31bd3b1841e4ed0b3539f6f55943
7
+ data.tar.gz: 6170f58d6f984e9ea7ac87f9bc4b71e7e69e6274d1b848998dd3c53ebb2de6bf6e50fa6246553ce513f028698561f7c93816f8ea2b6f224a2ca02acb10beac0e
data/.gitignore CHANGED
@@ -2,3 +2,5 @@
2
2
  /tmp
3
3
  .byebug_history
4
4
  /coverage
5
+
6
+ /Gemfile.lock
data/.rubocop.yml CHANGED
@@ -1,18 +1,20 @@
1
- require:
1
+ ---
2
+
3
+ plugins:
2
4
  - rubocop-rake
3
5
 
4
6
  AllCops:
5
- TargetRubyVersion: 2.4
7
+ TargetRubyVersion: 2.7
6
8
  NewCops: enable
7
9
 
8
10
  Gemspec/DevelopmentDependencies:
9
- Enabled: false
11
+ EnforcedStyle: gemspec
10
12
 
11
13
  Layout/LineLength:
12
14
  Max: 100
13
15
 
14
- # Metrics/AbcSize:
15
- # Max: 40
16
+ Metrics/AbcSize:
17
+ Max: 25
16
18
 
17
19
  Metrics/BlockLength:
18
20
  Exclude:
data/README.md CHANGED
@@ -2,25 +2,31 @@
2
2
 
3
3
  [Fluentd](https://fluentd.org/) plugin to dmux records.
4
4
 
5
- ## plugins
5
+ ## plugin - out - record_dmux
6
6
 
7
- ### out - record_dmux
7
+ ### config
8
8
 
9
- #### config
9
+ | setting | type | default | description |
10
+ |----------------------|-----------------|---------|----------------------------------------------------------------------------|
11
+ | tag | string | *nil* | tag to emit records on |
12
+ | demux_keys | array of string | *nil* | keys to demux, computed when nil |
13
+ | shared_keys | array of string | [] | keys to not demux, and to keep |
14
+ | remove_keys | array of string | [] | keys to remove |
15
+ | event_key_uniformize | bool | false | change record/event structure to name/value |
16
+ | event_key_prefix | string | '' | prefix before *name* in case of event_key_uniformize |
17
+ | shared_key_prefix | string | '' | prefix to add in from of *shared keys* key name |
18
+ | timestamp_key | string | *nil* | add timestamp key when defined, computed from time of the event |
19
+ | timestamp_format | enum | iso | can be iso for iso8601(3) or epochmillis for epoch with millisecond format |
10
20
 
11
- | setting | type | default | description |
12
- |---------|------|---------|-------------|
13
- | | | | |
14
-
15
- #### example
21
+ ### example
16
22
 
17
23
  Example of configuration
18
24
 
19
25
  ``` text
20
26
  <match *>
21
- @type record_dmux
27
+ @type record_demux
22
28
 
23
- tag data.dmux
29
+ tag data.demux
24
30
 
25
31
  demux_keys a, b, c
26
32
  shared_keys d, e, f
@@ -42,6 +48,6 @@ Add to Gemfile with:
42
48
 
43
49
  ## Copyright
44
50
 
45
- * Copyright(c) 2024- Thomas Tych
51
+ * Copyright(c) 2024-2025 Thomas Tych
46
52
  * License
47
53
  * Apache License, Version 2.0
data/fluent-plugin-record-demux.gemspec CHANGED
@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
5
 
6
6
  Gem::Specification.new do |spec|
7
7
  spec.name = 'fluent-plugin-record-demux'
8
- spec.version = '0.1.0'
8
+ spec.version = '0.1.1'
9
9
  spec.authors = ['Thomas Tych']
10
10
  spec.email = ['thomas.tych@gmail.com']
11
11
 
@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
13
13
  spec.homepage = 'https://gitlab.com/ttych/fluent-plugin-record-demux'
14
14
  spec.license = 'Apache-2.0'
15
15
 
16
- spec.required_ruby_version = '>= 2.4.0'
16
+ spec.required_ruby_version = '>= 2.7.0'
17
17
 
18
18
  spec.metadata['rubygems_mfa_required'] = 'true'
19
19
 
@@ -24,16 +24,16 @@ Gem::Specification.new do |spec|
24
24
  spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
25
25
  spec.require_paths = ['lib']
26
26
 
27
- spec.add_development_dependency 'bump', '~> 0.10.0'
28
- spec.add_development_dependency 'bundler', '~> 2.5.6'
27
+ spec.add_development_dependency 'bump', '~> 0.10'
28
+ spec.add_development_dependency 'bundler', '~> 2.6', '>= 2.6.5'
29
29
  spec.add_development_dependency 'byebug', '~> 11.1', '>= 11.1.3'
30
- spec.add_development_dependency 'rake', '~> 13.1.0'
31
- spec.add_development_dependency 'reek', '~> 6.1', '>= 6.1.4'
32
- spec.add_development_dependency 'rubocop', '~> 1.56'
33
- spec.add_development_dependency 'rubocop-rake', '~> 0.6.0'
34
- spec.add_development_dependency 'simplecov', '~> 0.22.0'
35
- spec.add_development_dependency 'test-unit', '~> 3.6.2'
36
- spec.add_development_dependency 'timecop', '~> 0.9.6'
37
-
38
- spec.add_runtime_dependency 'fluentd', ['>= 0.14.10', '< 2']
30
+ spec.add_development_dependency 'rake', '~> 13.2', '>= 13.2.1'
31
+ spec.add_development_dependency 'reek', '~> 6.4'
32
+ spec.add_development_dependency 'rubocop', '~> 1.74'
33
+ spec.add_development_dependency 'rubocop-rake', '~> 0.7', '>= 0.7.1'
34
+ spec.add_development_dependency 'simplecov', '~> 0.22'
35
+ spec.add_development_dependency 'test-unit', '~> 3.6', '>= 3.6.7'
36
+ spec.add_development_dependency 'timecop', '~> 0.9', '>= 0.9.10'
37
+
38
+ spec.add_dependency 'fluentd', ['>= 0.14.10', '< 2']
39
39
  end
data/lib/fluent/plugin/out_record_demux.rb CHANGED
@@ -29,11 +29,11 @@ module Fluent
29
29
  config_param :tag, :string, default: nil
30
30
 
31
31
  desc 'list of keys to demux'
32
- config_param :demux_keys, :array, default: nil
32
+ config_param :demux_keys, :array, value_type: :string, default: nil
33
33
  desc 'list of keys to be shared in all new records'
34
- config_param :shared_keys, :array, default: []
34
+ config_param :shared_keys, :array, value_type: :string, default: []
35
35
  desc 'list of keys to be removed'
36
- config_param :remove_keys, :array, default: []
36
+ config_param :remove_keys, :array, value_type: :string, default: []
37
37
 
38
38
  desc 'event key format uniformize'
39
39
  config_param :event_key_uniformize, :bool, default: false
@@ -53,29 +53,29 @@ module Fluent
53
53
 
54
54
  return unless @tag.nil?
55
55
 
56
- raise Fluent::ConfigError, 'out_record_splitter: `tag` must be specified'
56
+ raise Fluent::ConfigError, "#{NAME}: `tag` must be specified"
57
57
  end
58
58
 
59
59
  def multi_workers_ready?
60
60
  true
61
61
  end
62
62
 
63
- def process(_tag, es)
64
- es.each do |time, record|
63
+ def process(_events_tag, events)
64
+ demux_events = Fluent::EventStream.new
65
+ events.each do |time, record|
65
66
  record_keys = record.keys - remove_keys
67
+ record_shared_keys = record_keys.intersection(shared_keys)
68
+ record.slice(*record_shared_keys)
69
+ record_demux_keys = record_keys - record_shared_keys if !demux_keys || demux_keys.empty?
66
70
 
67
- shared_keys = record_keys.intersection(@shared_keys)
68
- shared = record.select { |key, _value| shared_keys.include?(key) }
69
- demux_keys = record_keys - shared_keys if !demux_keys || demux_keys.empty?
70
-
71
- demux_keys.each do |key|
71
+ record_demux_keys.each do |key|
72
72
  next unless record.key?(key)
73
73
 
74
74
  new_record = format(time, key, record[key], shared)
75
-
76
- router.emit(@tag, time, new_record)
75
+ demux_events.add(time, new_record)
77
76
  end
78
77
  end
78
+ router.emit_stream(tag, demux_events)
79
79
  end
80
80
 
81
81
  private
metadata CHANGED
@@ -1,14 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-record-demux
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.1.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Thomas Tych
8
- autorequire:
9
8
  bindir: bin
10
9
  cert_chain: []
11
- date: 2024-04-15 00:00:00.000000000 Z
10
+ date: 2025-03-14 00:00:00.000000000 Z
12
11
  dependencies:
13
12
  - !ruby/object:Gem::Dependency
14
13
  name: bump
@@ -16,28 +15,34 @@ dependencies:
16
15
  requirements:
17
16
  - - "~>"
18
17
  - !ruby/object:Gem::Version
19
- version: 0.10.0
18
+ version: '0.10'
20
19
  type: :development
21
20
  prerelease: false
22
21
  version_requirements: !ruby/object:Gem::Requirement
23
22
  requirements:
24
23
  - - "~>"
25
24
  - !ruby/object:Gem::Version
26
- version: 0.10.0
25
+ version: '0.10'
27
26
  - !ruby/object:Gem::Dependency
28
27
  name: bundler
29
28
  requirement: !ruby/object:Gem::Requirement
30
29
  requirements:
31
30
  - - "~>"
32
31
  - !ruby/object:Gem::Version
33
- version: 2.5.6
32
+ version: '2.6'
33
+ - - ">="
34
+ - !ruby/object:Gem::Version
35
+ version: 2.6.5
34
36
  type: :development
35
37
  prerelease: false
36
38
  version_requirements: !ruby/object:Gem::Requirement
37
39
  requirements:
38
40
  - - "~>"
39
41
  - !ruby/object:Gem::Version
40
- version: 2.5.6
42
+ version: '2.6'
43
+ - - ">="
44
+ - !ruby/object:Gem::Version
45
+ version: 2.6.5
41
46
  - !ruby/object:Gem::Dependency
42
47
  name: byebug
43
48
  requirement: !ruby/object:Gem::Requirement
@@ -64,104 +69,122 @@ dependencies:
64
69
  requirements:
65
70
  - - "~>"
66
71
  - !ruby/object:Gem::Version
67
- version: 13.1.0
72
+ version: '13.2'
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: 13.2.1
68
76
  type: :development
69
77
  prerelease: false
70
78
  version_requirements: !ruby/object:Gem::Requirement
71
79
  requirements:
72
80
  - - "~>"
73
81
  - !ruby/object:Gem::Version
74
- version: 13.1.0
82
+ version: '13.2'
83
+ - - ">="
84
+ - !ruby/object:Gem::Version
85
+ version: 13.2.1
75
86
  - !ruby/object:Gem::Dependency
76
87
  name: reek
77
88
  requirement: !ruby/object:Gem::Requirement
78
89
  requirements:
79
90
  - - "~>"
80
91
  - !ruby/object:Gem::Version
81
- version: '6.1'
82
- - - ">="
83
- - !ruby/object:Gem::Version
84
- version: 6.1.4
92
+ version: '6.4'
85
93
  type: :development
86
94
  prerelease: false
87
95
  version_requirements: !ruby/object:Gem::Requirement
88
96
  requirements:
89
97
  - - "~>"
90
98
  - !ruby/object:Gem::Version
91
- version: '6.1'
92
- - - ">="
93
- - !ruby/object:Gem::Version
94
- version: 6.1.4
99
+ version: '6.4'
95
100
  - !ruby/object:Gem::Dependency
96
101
  name: rubocop
97
102
  requirement: !ruby/object:Gem::Requirement
98
103
  requirements:
99
104
  - - "~>"
100
105
  - !ruby/object:Gem::Version
101
- version: '1.56'
106
+ version: '1.74'
102
107
  type: :development
103
108
  prerelease: false
104
109
  version_requirements: !ruby/object:Gem::Requirement
105
110
  requirements:
106
111
  - - "~>"
107
112
  - !ruby/object:Gem::Version
108
- version: '1.56'
113
+ version: '1.74'
109
114
  - !ruby/object:Gem::Dependency
110
115
  name: rubocop-rake
111
116
  requirement: !ruby/object:Gem::Requirement
112
117
  requirements:
113
118
  - - "~>"
114
119
  - !ruby/object:Gem::Version
115
- version: 0.6.0
120
+ version: '0.7'
121
+ - - ">="
122
+ - !ruby/object:Gem::Version
123
+ version: 0.7.1
116
124
  type: :development
117
125
  prerelease: false
118
126
  version_requirements: !ruby/object:Gem::Requirement
119
127
  requirements:
120
128
  - - "~>"
121
129
  - !ruby/object:Gem::Version
122
- version: 0.6.0
130
+ version: '0.7'
131
+ - - ">="
132
+ - !ruby/object:Gem::Version
133
+ version: 0.7.1
123
134
  - !ruby/object:Gem::Dependency
124
135
  name: simplecov
125
136
  requirement: !ruby/object:Gem::Requirement
126
137
  requirements:
127
138
  - - "~>"
128
139
  - !ruby/object:Gem::Version
129
- version: 0.22.0
140
+ version: '0.22'
130
141
  type: :development
131
142
  prerelease: false
132
143
  version_requirements: !ruby/object:Gem::Requirement
133
144
  requirements:
134
145
  - - "~>"
135
146
  - !ruby/object:Gem::Version
136
- version: 0.22.0
147
+ version: '0.22'
137
148
  - !ruby/object:Gem::Dependency
138
149
  name: test-unit
139
150
  requirement: !ruby/object:Gem::Requirement
140
151
  requirements:
141
152
  - - "~>"
142
153
  - !ruby/object:Gem::Version
143
- version: 3.6.2
154
+ version: '3.6'
155
+ - - ">="
156
+ - !ruby/object:Gem::Version
157
+ version: 3.6.7
144
158
  type: :development
145
159
  prerelease: false
146
160
  version_requirements: !ruby/object:Gem::Requirement
147
161
  requirements:
148
162
  - - "~>"
149
163
  - !ruby/object:Gem::Version
150
- version: 3.6.2
164
+ version: '3.6'
165
+ - - ">="
166
+ - !ruby/object:Gem::Version
167
+ version: 3.6.7
151
168
  - !ruby/object:Gem::Dependency
152
169
  name: timecop
153
170
  requirement: !ruby/object:Gem::Requirement
154
171
  requirements:
155
172
  - - "~>"
156
173
  - !ruby/object:Gem::Version
157
- version: 0.9.6
174
+ version: '0.9'
175
+ - - ">="
176
+ - !ruby/object:Gem::Version
177
+ version: 0.9.10
158
178
  type: :development
159
179
  prerelease: false
160
180
  version_requirements: !ruby/object:Gem::Requirement
161
181
  requirements:
162
182
  - - "~>"
163
183
  - !ruby/object:Gem::Version
164
- version: 0.9.6
184
+ version: '0.9'
185
+ - - ">="
186
+ - !ruby/object:Gem::Version
187
+ version: 0.9.10
165
188
  - !ruby/object:Gem::Dependency
166
189
  name: fluentd
167
190
  requirement: !ruby/object:Gem::Requirement
@@ -182,7 +205,6 @@ dependencies:
182
205
  - - "<"
183
206
  - !ruby/object:Gem::Version
184
207
  version: '2'
185
- description:
186
208
  email:
187
209
  - thomas.tych@gmail.com
188
210
  executables: []
@@ -192,7 +214,6 @@ files:
192
214
  - ".gitignore"
193
215
  - ".rubocop.yml"
194
216
  - Gemfile
195
- - Gemfile.lock
196
217
  - LICENSE
197
218
  - README.md
198
219
  - Rakefile
@@ -203,7 +224,6 @@ licenses:
203
224
  - Apache-2.0
204
225
  metadata:
205
226
  rubygems_mfa_required: 'true'
206
- post_install_message:
207
227
  rdoc_options: []
208
228
  require_paths:
209
229
  - lib
@@ -211,15 +231,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
211
231
  requirements:
212
232
  - - ">="
213
233
  - !ruby/object:Gem::Version
214
- version: 2.4.0
234
+ version: 2.7.0
215
235
  required_rubygems_version: !ruby/object:Gem::Requirement
216
236
  requirements:
217
237
  - - ">="
218
238
  - !ruby/object:Gem::Version
219
239
  version: '0'
220
240
  requirements: []
221
- rubygems_version: 3.5.6
222
- signing_key:
241
+ rubygems_version: 3.6.5
223
242
  specification_version: 4
224
243
  summary: fluentd plugin to demux record by keys.
225
244
  test_files: []
data/Gemfile.lock DELETED
@@ -1,131 +0,0 @@
1
- PATH
2
- remote: .
3
- specs:
4
- fluent-plugin-record-demux (0.1.0)
5
- fluentd (>= 0.14.10, < 2)
6
-
7
- GEM
8
- remote: https://rubygems.org/
9
- specs:
10
- ast (2.4.2)
11
- bigdecimal (3.1.7)
12
- bump (0.10.0)
13
- byebug (11.1.3)
14
- concurrent-ruby (1.2.3)
15
- cool.io (1.8.0)
16
- docile (1.4.0)
17
- dry-configurable (1.1.0)
18
- dry-core (~> 1.0, < 2)
19
- zeitwerk (~> 2.6)
20
- dry-core (1.0.1)
21
- concurrent-ruby (~> 1.0)
22
- zeitwerk (~> 2.6)
23
- dry-inflector (1.0.0)
24
- dry-initializer (3.1.1)
25
- dry-logic (1.5.0)
26
- concurrent-ruby (~> 1.0)
27
- dry-core (~> 1.0, < 2)
28
- zeitwerk (~> 2.6)
29
- dry-schema (1.13.3)
30
- concurrent-ruby (~> 1.0)
31
- dry-configurable (~> 1.0, >= 1.0.1)
32
- dry-core (~> 1.0, < 2)
33
- dry-initializer (~> 3.0)
34
- dry-logic (>= 1.4, < 2)
35
- dry-types (>= 1.7, < 2)
36
- zeitwerk (~> 2.6)
37
- dry-types (1.7.2)
38
- bigdecimal (~> 3.0)
39
- concurrent-ruby (~> 1.0)
40
- dry-core (~> 1.0)
41
- dry-inflector (~> 1.0)
42
- dry-logic (~> 1.4)
43
- zeitwerk (~> 2.6)
44
- fluentd (1.16.5)
45
- bundler
46
- cool.io (>= 1.4.5, < 2.0.0)
47
- http_parser.rb (>= 0.5.1, < 0.9.0)
48
- msgpack (>= 1.3.1, < 2.0.0)
49
- serverengine (>= 2.3.2, < 3.0.0)
50
- sigdump (~> 0.2.5)
51
- strptime (>= 0.2.4, < 1.0.0)
52
- tzinfo (>= 1.0, < 3.0)
53
- tzinfo-data (~> 1.0)
54
- webrick (~> 1.4)
55
- yajl-ruby (~> 1.0)
56
- http_parser.rb (0.8.0)
57
- json (2.7.2)
58
- language_server-protocol (3.17.0.3)
59
- msgpack (1.7.2)
60
- parallel (1.24.0)
61
- parser (3.3.0.5)
62
- ast (~> 2.4.1)
63
- racc
64
- power_assert (2.0.3)
65
- racc (1.7.3)
66
- rainbow (3.1.1)
67
- rake (13.1.0)
68
- reek (6.3.0)
69
- dry-schema (~> 1.13.0)
70
- parser (~> 3.3.0)
71
- rainbow (>= 2.0, < 4.0)
72
- rexml (~> 3.1)
73
- regexp_parser (2.9.0)
74
- rexml (3.2.6)
75
- rubocop (1.63.1)
76
- json (~> 2.3)
77
- language_server-protocol (>= 3.17.0)
78
- parallel (~> 1.10)
79
- parser (>= 3.3.0.2)
80
- rainbow (>= 2.2.2, < 4.0)
81
- regexp_parser (>= 1.8, < 3.0)
82
- rexml (>= 3.2.5, < 4.0)
83
- rubocop-ast (>= 1.31.1, < 2.0)
84
- ruby-progressbar (~> 1.7)
85
- unicode-display_width (>= 2.4.0, < 3.0)
86
- rubocop-ast (1.31.2)
87
- parser (>= 3.3.0.4)
88
- rubocop-rake (0.6.0)
89
- rubocop (~> 1.0)
90
- ruby-progressbar (1.13.0)
91
- serverengine (2.3.2)
92
- sigdump (~> 0.2.2)
93
- sigdump (0.2.5)
94
- simplecov (0.22.0)
95
- docile (~> 1.1)
96
- simplecov-html (~> 0.11)
97
- simplecov_json_formatter (~> 0.1)
98
- simplecov-html (0.12.3)
99
- simplecov_json_formatter (0.1.4)
100
- strptime (0.2.5)
101
- test-unit (3.6.2)
102
- power_assert
103
- timecop (0.9.8)
104
- tzinfo (2.0.6)
105
- concurrent-ruby (~> 1.0)
106
- tzinfo-data (1.2024.1)
107
- tzinfo (>= 1.0.0)
108
- unicode-display_width (2.5.0)
109
- webrick (1.8.1)
110
- yajl-ruby (1.4.3)
111
- zeitwerk (2.6.13)
112
-
113
- PLATFORMS
114
- ruby
115
- x86_64-linux
116
-
117
- DEPENDENCIES
118
- bump (~> 0.10.0)
119
- bundler (~> 2.5.6)
120
- byebug (~> 11.1, >= 11.1.3)
121
- fluent-plugin-record-demux!
122
- rake (~> 13.1.0)
123
- reek (~> 6.1, >= 6.1.4)
124
- rubocop (~> 1.56)
125
- rubocop-rake (~> 0.6.0)
126
- simplecov (~> 0.22.0)
127
- test-unit (~> 3.6.2)
128
- timecop (~> 0.9.6)
129
-
130
- BUNDLED WITH
131
- 2.5.6