fluent-plugin-avro_turf 0.1.0 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 43754b44341394535269181520729c165275940b1ac47ac92909ec3931224e88
4
- data.tar.gz: 3a7755a89407782b83f0f6bfb3e3cb302778f44655d7e2d69d5b7bdf23587aed
3
+ metadata.gz: 9b51ae67da9202930835719b7a5c4c3994856312650db05321649a9e498de7b2
4
+ data.tar.gz: 7cb372e88372dd44bfb85cd657d9f91892a194ccbc649d94477a52e02625afcd
5
5
  SHA512:
6
- metadata.gz: 7a66e6a8c854020b68dd6ea4062c949d4e8ae1c53e7e8b325812f679759687519034f5397fb3155143d25098e256eb587dfc0264131641074f98fd6daeaf913c
7
- data.tar.gz: 1d16e6a70d909610719e2222d36841cc4a39abd2b85066dd35f3d0c6bde23565373027ef98b75d0332349c821906f6a689f257eb0b06bc1d208cac73fdf745c5
6
+ metadata.gz: 13394038ac3b6e0180e08006203a6935e8431ca1ea812429cdfb7d5a4239ea5008012fac8ca76d403e0dae6d615ac2a1b65af021320842e94e989b624fc87618
7
+ data.tar.gz: 5828c306eb26c2effba7a34f3479563ff9ad5b8a53eda3e7f72470d727f993180f9e2af2db6f2bedbe9fbf354498747910c70dd07205a89a85832719024452b7
@@ -0,0 +1,27 @@
1
+ name: Test
2
+
3
+ on:
4
+ push:
5
+ branches: [ master ]
6
+ pull_request:
7
+
8
+ jobs:
9
+ test:
10
+
11
+ runs-on: ubuntu-latest
12
+ strategy:
13
+ matrix:
14
+ ruby-version: ['2.7', '3.0', '3.1']
15
+
16
+ steps:
17
+ - uses: actions/checkout@v3
18
+ - name: Set up Ruby
19
+ # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
20
+ # change this to (see https://github.com/ruby/setup-ruby#versioning):
21
+ # uses: ruby/setup-ruby@v1
22
+ uses: ruby/setup-ruby@v1
23
+ with:
24
+ ruby-version: ${{ matrix.ruby-version }}
25
+ bundler-cache: true # runs 'bundle install' and caches installed gems automatically
26
+ - name: Run tests
27
+ run: bundle exec rake
data/README.md CHANGED
@@ -54,12 +54,16 @@ Example.
54
54
  | default_schema_name | string | | nil | Default schema name when the record doesn't have schema_name_key |
55
55
  | schema_name_key | string | | `"schema_name"` | Field for schema name |
56
56
  | schema | hash | | nil | Inline schema definition. If this parameter is set, `default_schema_name` and `schema_name_key` are ignored |
57
+ | subject | string | | nil | Set subject explicitly |
58
+ | subject_key | string | | nil | Field for subject |
57
59
  | default_namespace | string | | nil | Default schema namespace |
58
60
  | namespace_key | string | | `"namespace"` | Field for namespace |
61
+ | schema_version | string | | `"latest"` | Set schema version explicitly that is only effective when subject or subject_key is set |
59
62
  | schema_version_key | string | | `"schema_version"` | Field for schema version |
60
63
  | exclude_schema_name_key | bool | | false | Set true to remove schema_name_key field from data |
61
64
  | exclude_namespace_key | bool | | false | Set true to remove namespace_key field from data |
62
65
  | exclude_schema_version_key | bool | | false | Set true to remove schema_version_key field from data |
66
+ | exclude_subject_key | bool | | false | Set true to remove subject_key field from data |
63
67
 
64
68
 
65
69
  ## Copyright
@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
3
 
4
4
  Gem::Specification.new do |spec|
5
5
  spec.name = "fluent-plugin-avro_turf"
6
- spec.version = "0.1.0"
6
+ spec.version = "0.2.0"
7
7
  spec.authors = ["joker1007"]
8
8
  spec.email = ["kakyoin.hierophant@gmail.com"]
9
9
 
@@ -30,20 +30,28 @@ module Fluent
30
30
  config_param :default_schema_name, :string, default: nil, desc: "Default schema name when the record doesn't have schema_name_key"
31
31
  config_param :schema_name_key, :string, default: "schema_name", desc: "Field for schema name"
32
32
 
33
+ config_param :subject, :string, default: nil, desc: "Set Subject explicitly"
34
+ config_param :subject_key, :string, default: nil, desc: "Field for subject (that is override the value that is set by `subject`)"
35
+
33
36
  config_param :schema, :hash, default: nil, desc: "Inline schema definition. If this parameter is set, `default_schema_name` and `schema_name_key` are ignored"
34
37
 
38
+
35
39
  config_param :default_namespace, :string, default: nil, desc: "Default schema namespace"
36
40
  config_param :namespace_key, :string, default: "namespace", desc: "Field for namespace"
37
41
 
38
- config_param :schema_version_key, :string, default: "schema_version", desc: "Field for schema version"
42
+ config_param :schema_version, :string, default: nil, desc: "Set schema_version explicitly"
43
+ config_param :schema_version_key, :string, default: nil, desc: "Field for schema version (that is override the value that is set by `schema_version`)"
39
44
 
40
45
  config_param :exclude_schema_name_key, :bool, default: false, desc: "Set true to remove schema_name_key field from data"
41
46
  config_param :exclude_namespace_key, :bool, default: false, desc: "Set true to remove namespace_key field from data"
42
47
  config_param :exclude_schema_version_key, :bool, default: false, desc: "Set true to remove schema_version_key field from data"
48
+ config_param :exclude_subject_key, :bool, default: false, desc: "Set true to remove subject_key field from data"
43
49
 
44
50
  def configure(conf)
45
51
  super
46
52
 
53
+ raise Fluent::ConfigError, "Do not set `schema` and `subject` at the same time" if @schema && @subject
54
+
47
55
  @avro_turf = AvroTurf::Messaging.new(registry_url: @schema_registry_url, schemas_path: @schemas_path)
48
56
  if @schema
49
57
  schema_store = @avro_turf.instance_variable_get("@schema_store")
@@ -52,6 +60,12 @@ module Fluent
52
60
  raise AvroTurfVersionImcompatible.new("Cannot access @schemas in @schema_store") unless schemas
53
61
  Avro::Schema.real_parse(@schema, schemas)
54
62
  end
63
+
64
+ if @subject && @schema_version.nil?
65
+ # use latest version when fluentd is launced
66
+ @schema_version = @avro_turf.instance_variable_get("@registry").subject_versions(@subject).last
67
+
68
+ end
55
69
  end
56
70
 
57
71
  def format(tag, time, record)
@@ -70,11 +84,17 @@ module Fluent
70
84
  end
71
85
  namespace ||= @default_namespace
72
86
 
87
+ if @subject_key
88
+ subject = @exclude_subject_key ? record.delete(@subject_key) : record[@subject_key]
89
+ end
90
+ subject ||= @subject
91
+
73
92
  if @schema_version_key
74
93
  schema_version = @exclude_schema_version_key ? record.delete(@schema_version_key) : record[@schema_version_key]
75
94
  end
95
+ schema_version ||= @schema_version
76
96
 
77
- @avro_turf.encode(record, schema_name: schema_name, namespace: namespace, version: schema_version)
97
+ @avro_turf.encode(record, schema_name: schema_name, namespace: namespace, subject: subject, version: schema_version)
78
98
  end
79
99
  end
80
100
  end
@@ -32,6 +32,15 @@ class AvroTurfFormatterTest < Test::Unit::TestCase
32
32
  }
33
33
  end
34
34
 
35
+ test "config error" do
36
+ assert_raise(Fluent::ConfigError) do
37
+ create_driver(
38
+ schema: "dummy",
39
+ subject: "dummy",
40
+ )
41
+ end
42
+ end
43
+
35
44
  test "format" do
36
45
  d = create_driver(
37
46
  schema_registry_url: registry_url,
@@ -43,6 +52,19 @@ class AvroTurfFormatterTest < Test::Unit::TestCase
43
52
  assert { decoded == user_record }
44
53
  end
45
54
 
55
+ test "format with subject" do
56
+ schema = File.read(File.join(schemas_path, "user.avsc"))
57
+ AvroTurf::ConfluentSchemaRegistry.new(registry_url).register("users-value", schema)
58
+ d = create_driver(
59
+ schema_registry_url: registry_url,
60
+ subject: "users-value",
61
+ )
62
+ formatted = d.instance.format("tag.test", event_time, user_record)
63
+ avro_turf = AvroTurf::Messaging.new(registry_url: registry_url, schemas_path: schemas_path)
64
+ decoded = avro_turf.decode(formatted, schema_name: "user")
65
+ assert { decoded == user_record }
66
+ end
67
+
46
68
  test "format with inline schema" do
47
69
  schema = {
48
70
  "name" => "person",
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-avro_turf
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - joker1007
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-08-24 00:00:00.000000000 Z
11
+ date: 2023-04-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -121,6 +121,7 @@ executables: []
121
121
  extensions: []
122
122
  extra_rdoc_files: []
123
123
  files:
124
+ - ".github/workflows/test.yml"
124
125
  - ".gitignore"
125
126
  - Gemfile
126
127
  - LICENSE
@@ -135,7 +136,7 @@ homepage: https://github.com/joker1007/fluent-plugin-avro_turf
135
136
  licenses:
136
137
  - Apache-2.0
137
138
  metadata: {}
138
- post_install_message:
139
+ post_install_message:
139
140
  rdoc_options: []
140
141
  require_paths:
141
142
  - lib
@@ -150,8 +151,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
150
151
  - !ruby/object:Gem::Version
151
152
  version: '0'
152
153
  requirements: []
153
- rubygems_version: 3.0.3
154
- signing_key:
154
+ rubygems_version: 3.4.10
155
+ signing_key:
155
156
  specification_version: 4
156
157
  summary: Fluentd formatter plugin by avro_turf
157
158
  test_files: