fluent-plugin-avro_turf 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 9b51ae67da9202930835719b7a5c4c3994856312650db05321649a9e498de7b2
|
|
4
|
+
data.tar.gz: 7cb372e88372dd44bfb85cd657d9f91892a194ccbc649d94477a52e02625afcd
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 13394038ac3b6e0180e08006203a6935e8431ca1ea812429cdfb7d5a4239ea5008012fac8ca76d403e0dae6d615ac2a1b65af021320842e94e989b624fc87618
|
|
7
|
+
data.tar.gz: 5828c306eb26c2effba7a34f3479563ff9ad5b8a53eda3e7f72470d727f993180f9e2af2db6f2bedbe9fbf354498747910c70dd07205a89a85832719024452b7
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
name: Test
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [ master ]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
strategy:
|
|
13
|
+
matrix:
|
|
14
|
+
ruby-version: ['2.7', '3.0', '3.1']
|
|
15
|
+
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v3
|
|
18
|
+
- name: Set up Ruby
|
|
19
|
+
# To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
|
|
20
|
+
# change this to (see https://github.com/ruby/setup-ruby#versioning):
|
|
21
|
+
# uses: ruby/setup-ruby@v1
|
|
22
|
+
uses: ruby/setup-ruby@v1
|
|
23
|
+
with:
|
|
24
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
25
|
+
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
|
|
26
|
+
- name: Run tests
|
|
27
|
+
run: bundle exec rake
|
data/README.md
CHANGED
|
@@ -54,12 +54,16 @@ Example.
|
|
|
54
54
|
| default_schema_name | string | | nil | Default schema name when the record doesn't have schema_name_key |
|
|
55
55
|
| schema_name_key | string | | `"schema_name"` | Field for schema name |
|
|
56
56
|
| schema | hash | | nil | Inline schema definition. If this parameter is set, `default_schema_name` and `schema_name_key` are ignored |
|
|
57
|
+
| subject | string | | nil | Set subject explicitly |
|
|
58
|
+
| subject_key | string | | nil | Field for subject |
|
|
57
59
|
| default_namespace | string | | nil | Default schema namespace |
|
|
58
60
|
| namespace_key | string | | `"namespace"` | Field for namespace |
|
|
61
|
+
| schema_version | string | | `"latest"` | Set schema version explicitly that is only effective when subject or subject_key is set |
|
|
59
62
|
| schema_version_key | string | | `"schema_version"` | Field for schema version |
|
|
60
63
|
| exclude_schema_name_key | bool | | false | Set true to remove schema_name_key field from data |
|
|
61
64
|
| exclude_namespace_key | bool | | false | Set true to remove namespace_key field from data |
|
|
62
65
|
| exclude_schema_version_key | bool | | false | Set true to remove schema_version_key field from data |
|
|
66
|
+
| exclude_subject_key | bool | | false | Set true to remove subject_key field from data |
|
|
63
67
|
|
|
64
68
|
|
|
65
69
|
## Copyright
|
|
@@ -30,20 +30,28 @@ module Fluent
|
|
|
30
30
|
config_param :default_schema_name, :string, default: nil, desc: "Default schema name when the record doesn't have schema_name_key"
|
|
31
31
|
config_param :schema_name_key, :string, default: "schema_name", desc: "Field for schema name"
|
|
32
32
|
|
|
33
|
+
config_param :subject, :string, default: nil, desc: "Set Subject explicitly"
|
|
34
|
+
config_param :subject_key, :string, default: nil, desc: "Field for subject (that is override the value that is set by `subject`)"
|
|
35
|
+
|
|
33
36
|
config_param :schema, :hash, default: nil, desc: "Inline schema definition. If this parameter is set, `default_schema_name` and `schema_name_key` are ignored"
|
|
34
37
|
|
|
38
|
+
|
|
35
39
|
config_param :default_namespace, :string, default: nil, desc: "Default schema namespace"
|
|
36
40
|
config_param :namespace_key, :string, default: "namespace", desc: "Field for namespace"
|
|
37
41
|
|
|
38
|
-
config_param :
|
|
42
|
+
config_param :schema_version, :string, default: nil, desc: "Set schema_version explicitly"
|
|
43
|
+
config_param :schema_version_key, :string, default: nil, desc: "Field for schema version (that is override the value that is set by `schema_version`)"
|
|
39
44
|
|
|
40
45
|
config_param :exclude_schema_name_key, :bool, default: false, desc: "Set true to remove schema_name_key field from data"
|
|
41
46
|
config_param :exclude_namespace_key, :bool, default: false, desc: "Set true to remove namespace_key field from data"
|
|
42
47
|
config_param :exclude_schema_version_key, :bool, default: false, desc: "Set true to remove schema_version_key field from data"
|
|
48
|
+
config_param :exclude_subject_key, :bool, default: false, desc: "Set true to remove subject_key field from data"
|
|
43
49
|
|
|
44
50
|
def configure(conf)
|
|
45
51
|
super
|
|
46
52
|
|
|
53
|
+
raise Fluent::ConfigError, "Do not set `schema` and `subject` at the same time" if @schema && @subject
|
|
54
|
+
|
|
47
55
|
@avro_turf = AvroTurf::Messaging.new(registry_url: @schema_registry_url, schemas_path: @schemas_path)
|
|
48
56
|
if @schema
|
|
49
57
|
schema_store = @avro_turf.instance_variable_get("@schema_store")
|
|
@@ -52,6 +60,12 @@ module Fluent
|
|
|
52
60
|
raise AvroTurfVersionImcompatible.new("Cannot access @schemas in @schema_store") unless schemas
|
|
53
61
|
Avro::Schema.real_parse(@schema, schemas)
|
|
54
62
|
end
|
|
63
|
+
|
|
64
|
+
if @subject && @schema_version.nil?
|
|
65
|
+
# use latest version when fluentd is launced
|
|
66
|
+
@schema_version = @avro_turf.instance_variable_get("@registry").subject_versions(@subject).last
|
|
67
|
+
|
|
68
|
+
end
|
|
55
69
|
end
|
|
56
70
|
|
|
57
71
|
def format(tag, time, record)
|
|
@@ -70,11 +84,17 @@ module Fluent
|
|
|
70
84
|
end
|
|
71
85
|
namespace ||= @default_namespace
|
|
72
86
|
|
|
87
|
+
if @subject_key
|
|
88
|
+
subject = @exclude_subject_key ? record.delete(@subject_key) : record[@subject_key]
|
|
89
|
+
end
|
|
90
|
+
subject ||= @subject
|
|
91
|
+
|
|
73
92
|
if @schema_version_key
|
|
74
93
|
schema_version = @exclude_schema_version_key ? record.delete(@schema_version_key) : record[@schema_version_key]
|
|
75
94
|
end
|
|
95
|
+
schema_version ||= @schema_version
|
|
76
96
|
|
|
77
|
-
@avro_turf.encode(record, schema_name: schema_name, namespace: namespace, version: schema_version)
|
|
97
|
+
@avro_turf.encode(record, schema_name: schema_name, namespace: namespace, subject: subject, version: schema_version)
|
|
78
98
|
end
|
|
79
99
|
end
|
|
80
100
|
end
|
|
@@ -32,6 +32,15 @@ class AvroTurfFormatterTest < Test::Unit::TestCase
|
|
|
32
32
|
}
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
+
test "config error" do
|
|
36
|
+
assert_raise(Fluent::ConfigError) do
|
|
37
|
+
create_driver(
|
|
38
|
+
schema: "dummy",
|
|
39
|
+
subject: "dummy",
|
|
40
|
+
)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
35
44
|
test "format" do
|
|
36
45
|
d = create_driver(
|
|
37
46
|
schema_registry_url: registry_url,
|
|
@@ -43,6 +52,19 @@ class AvroTurfFormatterTest < Test::Unit::TestCase
|
|
|
43
52
|
assert { decoded == user_record }
|
|
44
53
|
end
|
|
45
54
|
|
|
55
|
+
test "format with subject" do
|
|
56
|
+
schema = File.read(File.join(schemas_path, "user.avsc"))
|
|
57
|
+
AvroTurf::ConfluentSchemaRegistry.new(registry_url).register("users-value", schema)
|
|
58
|
+
d = create_driver(
|
|
59
|
+
schema_registry_url: registry_url,
|
|
60
|
+
subject: "users-value",
|
|
61
|
+
)
|
|
62
|
+
formatted = d.instance.format("tag.test", event_time, user_record)
|
|
63
|
+
avro_turf = AvroTurf::Messaging.new(registry_url: registry_url, schemas_path: schemas_path)
|
|
64
|
+
decoded = avro_turf.decode(formatted, schema_name: "user")
|
|
65
|
+
assert { decoded == user_record }
|
|
66
|
+
end
|
|
67
|
+
|
|
46
68
|
test "format with inline schema" do
|
|
47
69
|
schema = {
|
|
48
70
|
"name" => "person",
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-avro_turf
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- joker1007
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2023-04-11 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -121,6 +121,7 @@ executables: []
|
|
|
121
121
|
extensions: []
|
|
122
122
|
extra_rdoc_files: []
|
|
123
123
|
files:
|
|
124
|
+
- ".github/workflows/test.yml"
|
|
124
125
|
- ".gitignore"
|
|
125
126
|
- Gemfile
|
|
126
127
|
- LICENSE
|
|
@@ -135,7 +136,7 @@ homepage: https://github.com/joker1007/fluent-plugin-avro_turf
|
|
|
135
136
|
licenses:
|
|
136
137
|
- Apache-2.0
|
|
137
138
|
metadata: {}
|
|
138
|
-
post_install_message:
|
|
139
|
+
post_install_message:
|
|
139
140
|
rdoc_options: []
|
|
140
141
|
require_paths:
|
|
141
142
|
- lib
|
|
@@ -150,8 +151,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
150
151
|
- !ruby/object:Gem::Version
|
|
151
152
|
version: '0'
|
|
152
153
|
requirements: []
|
|
153
|
-
rubygems_version: 3.
|
|
154
|
-
signing_key:
|
|
154
|
+
rubygems_version: 3.4.10
|
|
155
|
+
signing_key:
|
|
155
156
|
specification_version: 4
|
|
156
157
|
summary: Fluentd formatter plugin by avro_turf
|
|
157
158
|
test_files:
|