fluent-plugin-avro_turf 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 9b51ae67da9202930835719b7a5c4c3994856312650db05321649a9e498de7b2
|
|
4
|
+
data.tar.gz: 7cb372e88372dd44bfb85cd657d9f91892a194ccbc649d94477a52e02625afcd
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 13394038ac3b6e0180e08006203a6935e8431ca1ea812429cdfb7d5a4239ea5008012fac8ca76d403e0dae6d615ac2a1b65af021320842e94e989b624fc87618
|
|
7
|
+
data.tar.gz: 5828c306eb26c2effba7a34f3479563ff9ad5b8a53eda3e7f72470d727f993180f9e2af2db6f2bedbe9fbf354498747910c70dd07205a89a85832719024452b7
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
name: Test
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [ master ]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
strategy:
|
|
13
|
+
matrix:
|
|
14
|
+
ruby-version: ['2.7', '3.0', '3.1']
|
|
15
|
+
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v3
|
|
18
|
+
- name: Set up Ruby
|
|
19
|
+
# To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
|
|
20
|
+
# change this to (see https://github.com/ruby/setup-ruby#versioning):
|
|
21
|
+
# uses: ruby/setup-ruby@v1
|
|
22
|
+
uses: ruby/setup-ruby@v1
|
|
23
|
+
with:
|
|
24
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
25
|
+
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
|
|
26
|
+
- name: Run tests
|
|
27
|
+
run: bundle exec rake
|
data/README.md
CHANGED
|
@@ -54,12 +54,16 @@ Example.
|
|
|
54
54
|
| default_schema_name | string | | nil | Default schema name when the record doesn't have schema_name_key |
|
|
55
55
|
| schema_name_key | string | | `"schema_name"` | Field for schema name |
|
|
56
56
|
| schema | hash | | nil | Inline schema definition. If this parameter is set, `default_schema_name` and `schema_name_key` are ignored |
|
|
57
|
+
| subject | string | | nil | Set subject explicitly |
|
|
58
|
+
| subject_key | string | | nil | Field for subject |
|
|
57
59
|
| default_namespace | string | | nil | Default schema namespace |
|
|
58
60
|
| namespace_key | string | | `"namespace"` | Field for namespace |
|
|
61
|
+
| schema_version | string | | `"latest"` | Set schema version explicitly that is only effective when subject or subject_key is set |
|
|
59
62
|
| schema_version_key | string | | `"schema_version"` | Field for schema version |
|
|
60
63
|
| exclude_schema_name_key | bool | | false | Set true to remove schema_name_key field from data |
|
|
61
64
|
| exclude_namespace_key | bool | | false | Set true to remove namespace_key field from data |
|
|
62
65
|
| exclude_schema_version_key | bool | | false | Set true to remove schema_version_key field from data |
|
|
66
|
+
| exclude_subject_key | bool | | false | Set true to remove subject_key field from data |
|
|
63
67
|
|
|
64
68
|
|
|
65
69
|
## Copyright
|
|
@@ -30,20 +30,28 @@ module Fluent
|
|
|
30
30
|
config_param :default_schema_name, :string, default: nil, desc: "Default schema name when the record doesn't have schema_name_key"
|
|
31
31
|
config_param :schema_name_key, :string, default: "schema_name", desc: "Field for schema name"
|
|
32
32
|
|
|
33
|
+
config_param :subject, :string, default: nil, desc: "Set Subject explicitly"
|
|
34
|
+
config_param :subject_key, :string, default: nil, desc: "Field for subject (that is override the value that is set by `subject`)"
|
|
35
|
+
|
|
33
36
|
config_param :schema, :hash, default: nil, desc: "Inline schema definition. If this parameter is set, `default_schema_name` and `schema_name_key` are ignored"
|
|
34
37
|
|
|
38
|
+
|
|
35
39
|
config_param :default_namespace, :string, default: nil, desc: "Default schema namespace"
|
|
36
40
|
config_param :namespace_key, :string, default: "namespace", desc: "Field for namespace"
|
|
37
41
|
|
|
38
|
-
config_param :
|
|
42
|
+
config_param :schema_version, :string, default: nil, desc: "Set schema_version explicitly"
|
|
43
|
+
config_param :schema_version_key, :string, default: nil, desc: "Field for schema version (that is override the value that is set by `schema_version`)"
|
|
39
44
|
|
|
40
45
|
config_param :exclude_schema_name_key, :bool, default: false, desc: "Set true to remove schema_name_key field from data"
|
|
41
46
|
config_param :exclude_namespace_key, :bool, default: false, desc: "Set true to remove namespace_key field from data"
|
|
42
47
|
config_param :exclude_schema_version_key, :bool, default: false, desc: "Set true to remove schema_version_key field from data"
|
|
48
|
+
config_param :exclude_subject_key, :bool, default: false, desc: "Set true to remove subject_key field from data"
|
|
43
49
|
|
|
44
50
|
def configure(conf)
|
|
45
51
|
super
|
|
46
52
|
|
|
53
|
+
raise Fluent::ConfigError, "Do not set `schema` and `subject` at the same time" if @schema && @subject
|
|
54
|
+
|
|
47
55
|
@avro_turf = AvroTurf::Messaging.new(registry_url: @schema_registry_url, schemas_path: @schemas_path)
|
|
48
56
|
if @schema
|
|
49
57
|
schema_store = @avro_turf.instance_variable_get("@schema_store")
|
|
@@ -52,6 +60,12 @@ module Fluent
|
|
|
52
60
|
raise AvroTurfVersionImcompatible.new("Cannot access @schemas in @schema_store") unless schemas
|
|
53
61
|
Avro::Schema.real_parse(@schema, schemas)
|
|
54
62
|
end
|
|
63
|
+
|
|
64
|
+
if @subject && @schema_version.nil?
|
|
65
|
+
# use latest version when fluentd is launced
|
|
66
|
+
@schema_version = @avro_turf.instance_variable_get("@registry").subject_versions(@subject).last
|
|
67
|
+
|
|
68
|
+
end
|
|
55
69
|
end
|
|
56
70
|
|
|
57
71
|
def format(tag, time, record)
|
|
@@ -70,11 +84,17 @@ module Fluent
|
|
|
70
84
|
end
|
|
71
85
|
namespace ||= @default_namespace
|
|
72
86
|
|
|
87
|
+
if @subject_key
|
|
88
|
+
subject = @exclude_subject_key ? record.delete(@subject_key) : record[@subject_key]
|
|
89
|
+
end
|
|
90
|
+
subject ||= @subject
|
|
91
|
+
|
|
73
92
|
if @schema_version_key
|
|
74
93
|
schema_version = @exclude_schema_version_key ? record.delete(@schema_version_key) : record[@schema_version_key]
|
|
75
94
|
end
|
|
95
|
+
schema_version ||= @schema_version
|
|
76
96
|
|
|
77
|
-
@avro_turf.encode(record, schema_name: schema_name, namespace: namespace, version: schema_version)
|
|
97
|
+
@avro_turf.encode(record, schema_name: schema_name, namespace: namespace, subject: subject, version: schema_version)
|
|
78
98
|
end
|
|
79
99
|
end
|
|
80
100
|
end
|
|
@@ -32,6 +32,15 @@ class AvroTurfFormatterTest < Test::Unit::TestCase
|
|
|
32
32
|
}
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
+
test "config error" do
|
|
36
|
+
assert_raise(Fluent::ConfigError) do
|
|
37
|
+
create_driver(
|
|
38
|
+
schema: "dummy",
|
|
39
|
+
subject: "dummy",
|
|
40
|
+
)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
35
44
|
test "format" do
|
|
36
45
|
d = create_driver(
|
|
37
46
|
schema_registry_url: registry_url,
|
|
@@ -43,6 +52,19 @@ class AvroTurfFormatterTest < Test::Unit::TestCase
|
|
|
43
52
|
assert { decoded == user_record }
|
|
44
53
|
end
|
|
45
54
|
|
|
55
|
+
test "format with subject" do
|
|
56
|
+
schema = File.read(File.join(schemas_path, "user.avsc"))
|
|
57
|
+
AvroTurf::ConfluentSchemaRegistry.new(registry_url).register("users-value", schema)
|
|
58
|
+
d = create_driver(
|
|
59
|
+
schema_registry_url: registry_url,
|
|
60
|
+
subject: "users-value",
|
|
61
|
+
)
|
|
62
|
+
formatted = d.instance.format("tag.test", event_time, user_record)
|
|
63
|
+
avro_turf = AvroTurf::Messaging.new(registry_url: registry_url, schemas_path: schemas_path)
|
|
64
|
+
decoded = avro_turf.decode(formatted, schema_name: "user")
|
|
65
|
+
assert { decoded == user_record }
|
|
66
|
+
end
|
|
67
|
+
|
|
46
68
|
test "format with inline schema" do
|
|
47
69
|
schema = {
|
|
48
70
|
"name" => "person",
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-avro_turf
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- joker1007
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2023-04-11 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -121,6 +121,7 @@ executables: []
|
|
|
121
121
|
extensions: []
|
|
122
122
|
extra_rdoc_files: []
|
|
123
123
|
files:
|
|
124
|
+
- ".github/workflows/test.yml"
|
|
124
125
|
- ".gitignore"
|
|
125
126
|
- Gemfile
|
|
126
127
|
- LICENSE
|
|
@@ -135,7 +136,7 @@ homepage: https://github.com/joker1007/fluent-plugin-avro_turf
|
|
|
135
136
|
licenses:
|
|
136
137
|
- Apache-2.0
|
|
137
138
|
metadata: {}
|
|
138
|
-
post_install_message:
|
|
139
|
+
post_install_message:
|
|
139
140
|
rdoc_options: []
|
|
140
141
|
require_paths:
|
|
141
142
|
- lib
|
|
@@ -150,8 +151,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
150
151
|
- !ruby/object:Gem::Version
|
|
151
152
|
version: '0'
|
|
152
153
|
requirements: []
|
|
153
|
-
rubygems_version: 3.
|
|
154
|
-
signing_key:
|
|
154
|
+
rubygems_version: 3.4.10
|
|
155
|
+
signing_key:
|
|
155
156
|
specification_version: 4
|
|
156
157
|
summary: Fluentd formatter plugin by avro_turf
|
|
157
158
|
test_files:
|