fluent-plugin-parser-avro 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/linux.yml +1 -1
- data/.github/workflows/windows.yml +1 -1
- data/README.md +3 -0
- data/fluent-plugin-parser-avro.gemspec +3 -3
- data/lib/fluent/plugin/confluent_avro_schema_registry.rb +25 -3
- data/lib/fluent/plugin/parser_avro.rb +12 -2
- data/test/plugin/test_parser_avro.rb +64 -0
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b0329676a89e7150a2dbd1b276496d81585acf3a8eba105beacdfbe22260efbb
|
4
|
+
data.tar.gz: 7a32762b40f316238d3dc45a18741e14ffc307154b643936f766e0c69b979dfe
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2a97b4a1dac117a0ff810de6f45bb11131782a13af5739accdea57741fdf00d12d786a17ca7e54f5072517995f813216129e4d682e452a25813e9ca4da875d02
|
7
|
+
data.tar.gz: fd63a06ae328e8f8fd48f8be53f86a62d4b1085df411a8acfdb9a62fc1c80b943b3ee4b6d4272c7df7ac260e74f03ed308e4efce07c3ab9bb97adb5569da3918
|
data/.github/workflows/linux.yml
CHANGED
data/README.md
CHANGED
@@ -39,6 +39,8 @@ $ bundle
|
|
39
39
|
* **readers_schema_json** (string) (optional): avro schema definition hash for readers definition.
|
40
40
|
* **use_confluent_schema** (bool) (optional): Assume to use confluent schema. Confluent avro schema uses the first 5-bytes for magic byte (1 byte) and schema_id (4 bytes). This parameter specifies to skip reading the first 5-bytes or not.
|
41
41
|
* Default value: `true`.
|
42
|
+
* **api_key** (string) (optional): Set key for Basic authentication.
|
43
|
+
* **api_secret** (string) (optional): Set secret for Basic authentication.
|
42
44
|
|
43
45
|
### \<confluent_registry\> section (optional) (single)
|
44
46
|
|
@@ -76,6 +78,7 @@ Confluent AVRO schema registry should respond with REST API.
|
|
76
78
|
This plugin uses the following API:
|
77
79
|
|
78
80
|
* [`GET /subjects/(string: subject)/versions/(versionId: version)`](https://docs.confluent.io/current/schema-registry/develop/api.html#get--subjects-(string-%20subject)-versions)
|
81
|
+
* [`GET /schemas/ids/(int: id)`](https://docs.confluent.io/current/schema-registry/develop/api.html#get--schemas-ids-int-%20id)
|
79
82
|
|
80
83
|
Users can specify a URL for retrieving the latest schemna information with `<confluent_registry>`:
|
81
84
|
|
@@ -3,9 +3,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |spec|
|
5
5
|
spec.name = "fluent-plugin-parser-avro"
|
6
|
-
spec.version = "0.
|
7
|
-
spec.authors = ["Hiroshi Hatake"]
|
8
|
-
spec.email = ["cosmo0920.wp@gmail.com"]
|
6
|
+
spec.version = "0.3.0"
|
7
|
+
spec.authors = ["Hiroshi Hatake", "Kentaro Hayashi"]
|
8
|
+
spec.email = ["cosmo0920.wp@gmail.com", "kenhys@gmail.com"]
|
9
9
|
|
10
10
|
spec.summary = %q{Avro parser plugin for Fluentd}
|
11
11
|
spec.description = spec.summary
|
@@ -18,15 +18,19 @@ require "uri"
|
|
18
18
|
|
19
19
|
module Fluent
|
20
20
|
module Plugin
|
21
|
+
class ConfluentAvroSchemaRegistryUnauthorizedError < StandardError; end
|
22
|
+
|
21
23
|
class ConfluentAvroSchemaRegistry
|
22
|
-
def initialize(registry_url)
|
24
|
+
def initialize(registry_url, api_key=nil, api_secret=nil)
|
23
25
|
@registry_url = registry_url
|
26
|
+
@api_key = api_key
|
27
|
+
@api_secret = api_secret
|
24
28
|
end
|
25
29
|
|
26
30
|
def subject_version(subject, schema_key, version = "latest")
|
27
31
|
registry_uri = URI.parse(@registry_url)
|
28
32
|
registry_uri_with_versions = URI.join(registry_uri, "/subjects/#{subject}/versions/#{version}")
|
29
|
-
response =
|
33
|
+
response = get_response(registry_uri_with_versions)
|
30
34
|
if schema_key.nil?
|
31
35
|
response.body
|
32
36
|
else
|
@@ -37,13 +41,31 @@ module Fluent
|
|
37
41
|
def schema_with_id(schema_id, schema_key)
|
38
42
|
registry_uri = URI.parse(@registry_url)
|
39
43
|
registry_uri_with_ids = URI.join(registry_uri, "/schemas/ids/#{schema_id}")
|
40
|
-
response =
|
44
|
+
response = get_response(registry_uri_with_ids)
|
41
45
|
if schema_key.nil?
|
42
46
|
response.body
|
43
47
|
else
|
44
48
|
Yajl.load(response.body)[schema_key]
|
45
49
|
end
|
46
50
|
end
|
51
|
+
|
52
|
+
def get_response(uri)
|
53
|
+
response = if @api_key and @api_secret
|
54
|
+
Net::HTTP.start(uri.host, uri.port) do |http|
|
55
|
+
request = Net::HTTP::Get.new(uri.path)
|
56
|
+
request.basic_auth(@api_key, @api_secret)
|
57
|
+
http.request(request)
|
58
|
+
end
|
59
|
+
else
|
60
|
+
Net::HTTP.get_response(uri)
|
61
|
+
end
|
62
|
+
if @api_key and @api_secret
|
63
|
+
if response.is_a?(Net::HTTPUnauthorized)
|
64
|
+
raise ConfluentAvroSchemaRegistryUnauthorizedError
|
65
|
+
end
|
66
|
+
end
|
67
|
+
response
|
68
|
+
end
|
47
69
|
end
|
48
70
|
end
|
49
71
|
end
|
@@ -36,6 +36,8 @@ module Fluent
|
|
36
36
|
config_param :readers_schema_file, :string, :default => nil
|
37
37
|
config_param :readers_schema_json, :string, :default => nil
|
38
38
|
config_param :use_confluent_schema, :bool, :default => true
|
39
|
+
config_param :api_key, :string, :default => nil
|
40
|
+
config_param :api_secret, :string, :default => nil
|
39
41
|
config_section :confluent_registry, param_name: :avro_registry, required: false, multi: false do
|
40
42
|
config_param :url, :string
|
41
43
|
config_param :subject, :string
|
@@ -70,7 +72,7 @@ module Fluent
|
|
70
72
|
@readers_schema = Avro::Schema.parse(@readers_raw_schema)
|
71
73
|
@reader = Avro::IO::DatumReader.new(@writers_schema, @readers_schema)
|
72
74
|
elsif @avro_registry
|
73
|
-
@confluent_registry = Fluent::Plugin::ConfluentAvroSchemaRegistry.new(@avro_registry.url)
|
75
|
+
@confluent_registry = Fluent::Plugin::ConfluentAvroSchemaRegistry.new(@avro_registry.url, @api_key, @api_secret)
|
74
76
|
@raw_schema = @confluent_registry.subject_version(@avro_registry.subject,
|
75
77
|
@avro_registry.schema_key,
|
76
78
|
@avro_registry.schema_version)
|
@@ -162,7 +164,15 @@ module Fluent
|
|
162
164
|
|
163
165
|
def fetch_schema(url, schema_key)
|
164
166
|
uri = URI.parse(url)
|
165
|
-
response =
|
167
|
+
response = if @api_key and @api_secret
|
168
|
+
Net::HTTP.start(uri.host, uri.port) do |http|
|
169
|
+
request = Net::HTTP::Get.new(uri.path)
|
170
|
+
request.basic_auth(@api_key, @api_secret)
|
171
|
+
http.request(request)
|
172
|
+
end
|
173
|
+
else
|
174
|
+
Net::HTTP.get_response(uri)
|
175
|
+
end
|
166
176
|
if schema_key.nil?
|
167
177
|
response.body
|
168
178
|
else
|
@@ -450,6 +450,70 @@ class AvroParserTest < Test::Unit::TestCase
|
|
450
450
|
end
|
451
451
|
end
|
452
452
|
|
453
|
+
class AuthenticationTest < self
|
454
|
+
teardown do
|
455
|
+
@dummy_server_thread.kill
|
456
|
+
@dummy_server_thread.join
|
457
|
+
end
|
458
|
+
|
459
|
+
setup do
|
460
|
+
@got = []
|
461
|
+
@dummy_server_thread = Thread.new do
|
462
|
+
server = WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => AVRO_REGISTRY_PORT})
|
463
|
+
begin
|
464
|
+
htpasswd = WEBrick::HTTPAuth::Htpasswd.new('dot.htpasswd')
|
465
|
+
htpasswd.set_passwd(nil, "API_KEY", "API_SECRET")
|
466
|
+
authenticator = WEBrick::HTTPAuth::BasicAuth.new(:UserDB => htpasswd, :Realm => "")
|
467
|
+
server.mount_proc('/') do |req, res|
|
468
|
+
authenticator.authenticate(req, res)
|
469
|
+
schema = File.read(File.join(__dir__, "..", "data", "schema-persions-value-1.avsc"))
|
470
|
+
res.body = schema
|
471
|
+
end
|
472
|
+
server.start
|
473
|
+
ensure
|
474
|
+
server.shutdown
|
475
|
+
end
|
476
|
+
end
|
477
|
+
end
|
478
|
+
|
479
|
+
def test_authentication_failure
|
480
|
+
conf = Fluent::Config::Element.new(
|
481
|
+
'', '', {'@type' => 'avro',
|
482
|
+
'api_key' => 'WRONG_KEY',
|
483
|
+
'api_secret' => 'WRONG_SECRET'
|
484
|
+
}, [
|
485
|
+
Fluent::Config::Element.new('confluent_registry', '', {
|
486
|
+
'url' => 'http://localhost:8081',
|
487
|
+
'subject' => 'persons-avro-value',
|
488
|
+
}, [])
|
489
|
+
])
|
490
|
+
assert_raise(Fluent::Plugin::ConfluentAvroSchemaRegistryUnauthorizedError) do
|
491
|
+
d = create_driver(conf)
|
492
|
+
d.instance.run
|
493
|
+
end
|
494
|
+
end
|
495
|
+
|
496
|
+
def test_with_authentication
|
497
|
+
conf = Fluent::Config::Element.new(
|
498
|
+
'', '', {'@type' => 'avro',
|
499
|
+
'api_key' => 'API_KEY',
|
500
|
+
'api_secret' => 'API_SECRET'
|
501
|
+
}, [
|
502
|
+
Fluent::Config::Element.new('confluent_registry', '', {
|
503
|
+
'url' => 'http://localhost:8081',
|
504
|
+
'subject' => 'persons-avro-value',
|
505
|
+
}, [])
|
506
|
+
])
|
507
|
+
datum = {"firstName" => "Aleen","lastName" => "Terry","birthDate" => 159202477258}
|
508
|
+
schema = Yajl.load(File.read(File.join(__dir__, "..", "data", "schema-persions-value-1.avsc")))
|
509
|
+
encoded = encode_datum(datum, schema.fetch("schema"), true, 1)
|
510
|
+
d = create_driver(conf)
|
511
|
+
d.instance.parse(encoded) do |_time, record|
|
512
|
+
assert_equal datum, record
|
513
|
+
end
|
514
|
+
end
|
515
|
+
end
|
516
|
+
|
453
517
|
private
|
454
518
|
|
455
519
|
def encode_datum(datum, string_schema, use_confluent_schema = true, schema_id = 1)
|
metadata
CHANGED
@@ -1,14 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-parser-avro
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hiroshi Hatake
|
8
|
+
- Kentaro Hayashi
|
8
9
|
autorequire:
|
9
10
|
bindir: bin
|
10
11
|
cert_chain: []
|
11
|
-
date:
|
12
|
+
date: 2021-02-08 00:00:00.000000000 Z
|
12
13
|
dependencies:
|
13
14
|
- !ruby/object:Gem::Dependency
|
14
15
|
name: avro
|
@@ -89,6 +90,7 @@ dependencies:
|
|
89
90
|
description: Avro parser plugin for Fluentd
|
90
91
|
email:
|
91
92
|
- cosmo0920.wp@gmail.com
|
93
|
+
- kenhys@gmail.com
|
92
94
|
executables: []
|
93
95
|
extensions: []
|
94
96
|
extra_rdoc_files: []
|