fluent-plugin-parser-avro 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/linux.yml +1 -1
- data/.github/workflows/windows.yml +1 -1
- data/README.md +3 -0
- data/fluent-plugin-parser-avro.gemspec +3 -3
- data/lib/fluent/plugin/confluent_avro_schema_registry.rb +25 -3
- data/lib/fluent/plugin/parser_avro.rb +12 -2
- data/test/plugin/test_parser_avro.rb +64 -0
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b0329676a89e7150a2dbd1b276496d81585acf3a8eba105beacdfbe22260efbb
|
4
|
+
data.tar.gz: 7a32762b40f316238d3dc45a18741e14ffc307154b643936f766e0c69b979dfe
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2a97b4a1dac117a0ff810de6f45bb11131782a13af5739accdea57741fdf00d12d786a17ca7e54f5072517995f813216129e4d682e452a25813e9ca4da875d02
|
7
|
+
data.tar.gz: fd63a06ae328e8f8fd48f8be53f86a62d4b1085df411a8acfdb9a62fc1c80b943b3ee4b6d4272c7df7ac260e74f03ed308e4efce07c3ab9bb97adb5569da3918
|
data/.github/workflows/linux.yml
CHANGED
data/README.md
CHANGED
@@ -39,6 +39,8 @@ $ bundle
|
|
39
39
|
* **readers_schema_json** (string) (optional): avro schema definition hash for readers definition.
|
40
40
|
* **use_confluent_schema** (bool) (optional): Assume to use confluent schema. Confluent avro schema uses the first 5-bytes for magic byte (1 byte) and schema_id (4 bytes). This parameter specifies to skip reading the first 5-bytes or not.
|
41
41
|
* Default value: `true`.
|
42
|
+
* **api_key** (string) (optional): Set key for Basic authentication.
|
43
|
+
* **api_secret** (string) (optional): Set secret for Basic authentication.
|
42
44
|
|
43
45
|
### \<confluent_registry\> section (optional) (single)
|
44
46
|
|
@@ -76,6 +78,7 @@ Confluent AVRO schema registry should respond with REST API.
|
|
76
78
|
This plugin uses the following API:
|
77
79
|
|
78
80
|
* [`GET /subjects/(string: subject)/versions/(versionId: version)`](https://docs.confluent.io/current/schema-registry/develop/api.html#get--subjects-(string-%20subject)-versions)
|
81
|
+
* [`GET /schemas/ids/(int: id)`](https://docs.confluent.io/current/schema-registry/develop/api.html#get--schemas-ids-int-%20id)
|
79
82
|
|
80
83
|
Users can specify a URL for retrieving the latest schemna information with `<confluent_registry>`:
|
81
84
|
|
@@ -3,9 +3,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |spec|
|
5
5
|
spec.name = "fluent-plugin-parser-avro"
|
6
|
-
spec.version = "0.
|
7
|
-
spec.authors = ["Hiroshi Hatake"]
|
8
|
-
spec.email = ["cosmo0920.wp@gmail.com"]
|
6
|
+
spec.version = "0.3.0"
|
7
|
+
spec.authors = ["Hiroshi Hatake", "Kentaro Hayashi"]
|
8
|
+
spec.email = ["cosmo0920.wp@gmail.com", "kenhys@gmail.com"]
|
9
9
|
|
10
10
|
spec.summary = %q{Avro parser plugin for Fluentd}
|
11
11
|
spec.description = spec.summary
|
@@ -18,15 +18,19 @@ require "uri"
|
|
18
18
|
|
19
19
|
module Fluent
|
20
20
|
module Plugin
|
21
|
+
class ConfluentAvroSchemaRegistryUnauthorizedError < StandardError; end
|
22
|
+
|
21
23
|
class ConfluentAvroSchemaRegistry
|
22
|
-
def initialize(registry_url)
|
24
|
+
def initialize(registry_url, api_key=nil, api_secret=nil)
|
23
25
|
@registry_url = registry_url
|
26
|
+
@api_key = api_key
|
27
|
+
@api_secret = api_secret
|
24
28
|
end
|
25
29
|
|
26
30
|
def subject_version(subject, schema_key, version = "latest")
|
27
31
|
registry_uri = URI.parse(@registry_url)
|
28
32
|
registry_uri_with_versions = URI.join(registry_uri, "/subjects/#{subject}/versions/#{version}")
|
29
|
-
response =
|
33
|
+
response = get_response(registry_uri_with_versions)
|
30
34
|
if schema_key.nil?
|
31
35
|
response.body
|
32
36
|
else
|
@@ -37,13 +41,31 @@ module Fluent
|
|
37
41
|
def schema_with_id(schema_id, schema_key)
|
38
42
|
registry_uri = URI.parse(@registry_url)
|
39
43
|
registry_uri_with_ids = URI.join(registry_uri, "/schemas/ids/#{schema_id}")
|
40
|
-
response =
|
44
|
+
response = get_response(registry_uri_with_ids)
|
41
45
|
if schema_key.nil?
|
42
46
|
response.body
|
43
47
|
else
|
44
48
|
Yajl.load(response.body)[schema_key]
|
45
49
|
end
|
46
50
|
end
|
51
|
+
|
52
|
+
def get_response(uri)
|
53
|
+
response = if @api_key and @api_secret
|
54
|
+
Net::HTTP.start(uri.host, uri.port) do |http|
|
55
|
+
request = Net::HTTP::Get.new(uri.path)
|
56
|
+
request.basic_auth(@api_key, @api_secret)
|
57
|
+
http.request(request)
|
58
|
+
end
|
59
|
+
else
|
60
|
+
Net::HTTP.get_response(uri)
|
61
|
+
end
|
62
|
+
if @api_key and @api_secret
|
63
|
+
if response.is_a?(Net::HTTPUnauthorized)
|
64
|
+
raise ConfluentAvroSchemaRegistryUnauthorizedError
|
65
|
+
end
|
66
|
+
end
|
67
|
+
response
|
68
|
+
end
|
47
69
|
end
|
48
70
|
end
|
49
71
|
end
|
@@ -36,6 +36,8 @@ module Fluent
|
|
36
36
|
config_param :readers_schema_file, :string, :default => nil
|
37
37
|
config_param :readers_schema_json, :string, :default => nil
|
38
38
|
config_param :use_confluent_schema, :bool, :default => true
|
39
|
+
config_param :api_key, :string, :default => nil
|
40
|
+
config_param :api_secret, :string, :default => nil
|
39
41
|
config_section :confluent_registry, param_name: :avro_registry, required: false, multi: false do
|
40
42
|
config_param :url, :string
|
41
43
|
config_param :subject, :string
|
@@ -70,7 +72,7 @@ module Fluent
|
|
70
72
|
@readers_schema = Avro::Schema.parse(@readers_raw_schema)
|
71
73
|
@reader = Avro::IO::DatumReader.new(@writers_schema, @readers_schema)
|
72
74
|
elsif @avro_registry
|
73
|
-
@confluent_registry = Fluent::Plugin::ConfluentAvroSchemaRegistry.new(@avro_registry.url)
|
75
|
+
@confluent_registry = Fluent::Plugin::ConfluentAvroSchemaRegistry.new(@avro_registry.url, @api_key, @api_secret)
|
74
76
|
@raw_schema = @confluent_registry.subject_version(@avro_registry.subject,
|
75
77
|
@avro_registry.schema_key,
|
76
78
|
@avro_registry.schema_version)
|
@@ -162,7 +164,15 @@ module Fluent
|
|
162
164
|
|
163
165
|
def fetch_schema(url, schema_key)
|
164
166
|
uri = URI.parse(url)
|
165
|
-
response =
|
167
|
+
response = if @api_key and @api_secret
|
168
|
+
Net::HTTP.start(uri.host, uri.port) do |http|
|
169
|
+
request = Net::HTTP::Get.new(uri.path)
|
170
|
+
request.basic_auth(@api_key, @api_secret)
|
171
|
+
http.request(request)
|
172
|
+
end
|
173
|
+
else
|
174
|
+
Net::HTTP.get_response(uri)
|
175
|
+
end
|
166
176
|
if schema_key.nil?
|
167
177
|
response.body
|
168
178
|
else
|
@@ -450,6 +450,70 @@ class AvroParserTest < Test::Unit::TestCase
|
|
450
450
|
end
|
451
451
|
end
|
452
452
|
|
453
|
+
class AuthenticationTest < self
|
454
|
+
teardown do
|
455
|
+
@dummy_server_thread.kill
|
456
|
+
@dummy_server_thread.join
|
457
|
+
end
|
458
|
+
|
459
|
+
setup do
|
460
|
+
@got = []
|
461
|
+
@dummy_server_thread = Thread.new do
|
462
|
+
server = WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => AVRO_REGISTRY_PORT})
|
463
|
+
begin
|
464
|
+
htpasswd = WEBrick::HTTPAuth::Htpasswd.new('dot.htpasswd')
|
465
|
+
htpasswd.set_passwd(nil, "API_KEY", "API_SECRET")
|
466
|
+
authenticator = WEBrick::HTTPAuth::BasicAuth.new(:UserDB => htpasswd, :Realm => "")
|
467
|
+
server.mount_proc('/') do |req, res|
|
468
|
+
authenticator.authenticate(req, res)
|
469
|
+
schema = File.read(File.join(__dir__, "..", "data", "schema-persions-value-1.avsc"))
|
470
|
+
res.body = schema
|
471
|
+
end
|
472
|
+
server.start
|
473
|
+
ensure
|
474
|
+
server.shutdown
|
475
|
+
end
|
476
|
+
end
|
477
|
+
end
|
478
|
+
|
479
|
+
def test_authentication_failure
|
480
|
+
conf = Fluent::Config::Element.new(
|
481
|
+
'', '', {'@type' => 'avro',
|
482
|
+
'api_key' => 'WRONG_KEY',
|
483
|
+
'api_secret' => 'WRONG_SECRET'
|
484
|
+
}, [
|
485
|
+
Fluent::Config::Element.new('confluent_registry', '', {
|
486
|
+
'url' => 'http://localhost:8081',
|
487
|
+
'subject' => 'persons-avro-value',
|
488
|
+
}, [])
|
489
|
+
])
|
490
|
+
assert_raise(Fluent::Plugin::ConfluentAvroSchemaRegistryUnauthorizedError) do
|
491
|
+
d = create_driver(conf)
|
492
|
+
d.instance.run
|
493
|
+
end
|
494
|
+
end
|
495
|
+
|
496
|
+
def test_with_authentication
|
497
|
+
conf = Fluent::Config::Element.new(
|
498
|
+
'', '', {'@type' => 'avro',
|
499
|
+
'api_key' => 'API_KEY',
|
500
|
+
'api_secret' => 'API_SECRET'
|
501
|
+
}, [
|
502
|
+
Fluent::Config::Element.new('confluent_registry', '', {
|
503
|
+
'url' => 'http://localhost:8081',
|
504
|
+
'subject' => 'persons-avro-value',
|
505
|
+
}, [])
|
506
|
+
])
|
507
|
+
datum = {"firstName" => "Aleen","lastName" => "Terry","birthDate" => 159202477258}
|
508
|
+
schema = Yajl.load(File.read(File.join(__dir__, "..", "data", "schema-persions-value-1.avsc")))
|
509
|
+
encoded = encode_datum(datum, schema.fetch("schema"), true, 1)
|
510
|
+
d = create_driver(conf)
|
511
|
+
d.instance.parse(encoded) do |_time, record|
|
512
|
+
assert_equal datum, record
|
513
|
+
end
|
514
|
+
end
|
515
|
+
end
|
516
|
+
|
453
517
|
private
|
454
518
|
|
455
519
|
def encode_datum(datum, string_schema, use_confluent_schema = true, schema_id = 1)
|
metadata
CHANGED
@@ -1,14 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-parser-avro
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hiroshi Hatake
|
8
|
+
- Kentaro Hayashi
|
8
9
|
autorequire:
|
9
10
|
bindir: bin
|
10
11
|
cert_chain: []
|
11
|
-
date:
|
12
|
+
date: 2021-02-08 00:00:00.000000000 Z
|
12
13
|
dependencies:
|
13
14
|
- !ruby/object:Gem::Dependency
|
14
15
|
name: avro
|
@@ -89,6 +90,7 @@ dependencies:
|
|
89
90
|
description: Avro parser plugin for Fluentd
|
90
91
|
email:
|
91
92
|
- cosmo0920.wp@gmail.com
|
93
|
+
- kenhys@gmail.com
|
92
94
|
executables: []
|
93
95
|
extensions: []
|
94
96
|
extra_rdoc_files: []
|