fluent-plugin-parser-avro 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9633a316c1de1e4cb83b487d99d5a3814f1cde2b96d20633cfe728c584108761
4
- data.tar.gz: 6438fe5e248c98540602183a5828a7dbc596fe5000c4993fd428ee9223ed3077
3
+ metadata.gz: b0329676a89e7150a2dbd1b276496d81585acf3a8eba105beacdfbe22260efbb
4
+ data.tar.gz: 7a32762b40f316238d3dc45a18741e14ffc307154b643936f766e0c69b979dfe
5
5
  SHA512:
6
- metadata.gz: 8b8894c17aaa33916ef3601634d36139f2817db875eba2c5e552a0309f2c5f3ce6578526c5aad54244cd51ce36cf30bc04f64ce576fcbd4ccb89fc7726d4102d
7
- data.tar.gz: a9dbc9e144b1e0f7f2b585a51c0519e6c2953d7258b5bd13fd8d6246754f9956bffcee43777378692ea4ef2d8997d8f6648653d8928e2345e70c32e48ab849a6
6
+ metadata.gz: 2a97b4a1dac117a0ff810de6f45bb11131782a13af5739accdea57741fdf00d12d786a17ca7e54f5072517995f813216129e4d682e452a25813e9ca4da875d02
7
+ data.tar.gz: fd63a06ae328e8f8fd48f8be53f86a62d4b1085df411a8acfdb9a62fc1c80b943b3ee4b6d4272c7df7ac260e74f03ed308e4efce07c3ab9bb97adb5569da3918
@@ -8,7 +8,7 @@ jobs:
8
8
  strategy:
9
9
  fail-fast: false
10
10
  matrix:
11
- ruby: [ '2.4', '2.5', '2.6', '2.7' ]
11
+ ruby: [ '2.5', '2.6', '2.7' ]
12
12
  os:
13
13
  - ubuntu-latest
14
14
  name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
@@ -8,7 +8,7 @@ jobs:
8
8
  strategy:
9
9
  fail-fast: false
10
10
  matrix:
11
- ruby: [ '2.4', '2.5', '2.6', '2.7' ]
11
+ ruby: [ '2.5', '2.6', '2.7' ]
12
12
  os:
13
13
  - windows-latest
14
14
  name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
data/README.md CHANGED
@@ -39,6 +39,8 @@ $ bundle
39
39
  * **readers_schema_json** (string) (optional): avro schema definition hash for readers definition.
40
40
  * **use_confluent_schema** (bool) (optional): Assume to use confluent schema. Confluent avro schema uses the first 5-bytes for magic byte (1 byte) and schema_id (4 bytes). This parameter specifies to skip reading the first 5-bytes or not.
41
41
  * Default value: `true`.
42
+ * **api_key** (string) (optional): Set key for Basic authentication.
43
+ * **api_secret** (string) (optional): Set secret for Basic authentication.
42
44
 
43
45
  ### \<confluent_registry\> section (optional) (single)
44
46
 
@@ -76,6 +78,7 @@ Confluent AVRO schema registry should respond with REST API.
76
78
  This plugin uses the following API:
77
79
 
78
80
  * [`GET /subjects/(string: subject)/versions/(versionId: version)`](https://docs.confluent.io/current/schema-registry/develop/api.html#get--subjects-(string-%20subject)-versions)
81
+ * [`GET /schemas/ids/(int: id)`](https://docs.confluent.io/current/schema-registry/develop/api.html#get--schemas-ids-int-%20id)
79
82
 
80
83
  Users can specify a URL for retrieving the latest schemna information with `<confluent_registry>`:
81
84
 
@@ -3,9 +3,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
3
 
4
4
  Gem::Specification.new do |spec|
5
5
  spec.name = "fluent-plugin-parser-avro"
6
- spec.version = "0.2.0"
7
- spec.authors = ["Hiroshi Hatake"]
8
- spec.email = ["cosmo0920.wp@gmail.com"]
6
+ spec.version = "0.3.0"
7
+ spec.authors = ["Hiroshi Hatake", "Kentaro Hayashi"]
8
+ spec.email = ["cosmo0920.wp@gmail.com", "kenhys@gmail.com"]
9
9
 
10
10
  spec.summary = %q{Avro parser plugin for Fluentd}
11
11
  spec.description = spec.summary
@@ -18,15 +18,19 @@ require "uri"
18
18
 
19
19
  module Fluent
20
20
  module Plugin
21
+ class ConfluentAvroSchemaRegistryUnauthorizedError < StandardError; end
22
+
21
23
  class ConfluentAvroSchemaRegistry
22
- def initialize(registry_url)
24
+ def initialize(registry_url, api_key=nil, api_secret=nil)
23
25
  @registry_url = registry_url
26
+ @api_key = api_key
27
+ @api_secret = api_secret
24
28
  end
25
29
 
26
30
  def subject_version(subject, schema_key, version = "latest")
27
31
  registry_uri = URI.parse(@registry_url)
28
32
  registry_uri_with_versions = URI.join(registry_uri, "/subjects/#{subject}/versions/#{version}")
29
- response = Net::HTTP.get_response(registry_uri_with_versions)
33
+ response = get_response(registry_uri_with_versions)
30
34
  if schema_key.nil?
31
35
  response.body
32
36
  else
@@ -37,13 +41,31 @@ module Fluent
37
41
  def schema_with_id(schema_id, schema_key)
38
42
  registry_uri = URI.parse(@registry_url)
39
43
  registry_uri_with_ids = URI.join(registry_uri, "/schemas/ids/#{schema_id}")
40
- response = Net::HTTP.get_response(registry_uri_with_ids)
44
+ response = get_response(registry_uri_with_ids)
41
45
  if schema_key.nil?
42
46
  response.body
43
47
  else
44
48
  Yajl.load(response.body)[schema_key]
45
49
  end
46
50
  end
51
+
52
+ def get_response(uri)
53
+ response = if @api_key and @api_secret
54
+ Net::HTTP.start(uri.host, uri.port) do |http|
55
+ request = Net::HTTP::Get.new(uri.path)
56
+ request.basic_auth(@api_key, @api_secret)
57
+ http.request(request)
58
+ end
59
+ else
60
+ Net::HTTP.get_response(uri)
61
+ end
62
+ if @api_key and @api_secret
63
+ if response.is_a?(Net::HTTPUnauthorized)
64
+ raise ConfluentAvroSchemaRegistryUnauthorizedError
65
+ end
66
+ end
67
+ response
68
+ end
47
69
  end
48
70
  end
49
71
  end
@@ -36,6 +36,8 @@ module Fluent
36
36
  config_param :readers_schema_file, :string, :default => nil
37
37
  config_param :readers_schema_json, :string, :default => nil
38
38
  config_param :use_confluent_schema, :bool, :default => true
39
+ config_param :api_key, :string, :default => nil
40
+ config_param :api_secret, :string, :default => nil
39
41
  config_section :confluent_registry, param_name: :avro_registry, required: false, multi: false do
40
42
  config_param :url, :string
41
43
  config_param :subject, :string
@@ -70,7 +72,7 @@ module Fluent
70
72
  @readers_schema = Avro::Schema.parse(@readers_raw_schema)
71
73
  @reader = Avro::IO::DatumReader.new(@writers_schema, @readers_schema)
72
74
  elsif @avro_registry
73
- @confluent_registry = Fluent::Plugin::ConfluentAvroSchemaRegistry.new(@avro_registry.url)
75
+ @confluent_registry = Fluent::Plugin::ConfluentAvroSchemaRegistry.new(@avro_registry.url, @api_key, @api_secret)
74
76
  @raw_schema = @confluent_registry.subject_version(@avro_registry.subject,
75
77
  @avro_registry.schema_key,
76
78
  @avro_registry.schema_version)
@@ -162,7 +164,15 @@ module Fluent
162
164
 
163
165
  def fetch_schema(url, schema_key)
164
166
  uri = URI.parse(url)
165
- response = Net::HTTP.get_response(uri)
167
+ response = if @api_key and @api_secret
168
+ Net::HTTP.start(uri.host, uri.port) do |http|
169
+ request = Net::HTTP::Get.new(uri.path)
170
+ request.basic_auth(@api_key, @api_secret)
171
+ http.request(request)
172
+ end
173
+ else
174
+ Net::HTTP.get_response(uri)
175
+ end
166
176
  if schema_key.nil?
167
177
  response.body
168
178
  else
@@ -450,6 +450,70 @@ class AvroParserTest < Test::Unit::TestCase
450
450
  end
451
451
  end
452
452
 
453
+ class AuthenticationTest < self
454
+ teardown do
455
+ @dummy_server_thread.kill
456
+ @dummy_server_thread.join
457
+ end
458
+
459
+ setup do
460
+ @got = []
461
+ @dummy_server_thread = Thread.new do
462
+ server = WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => AVRO_REGISTRY_PORT})
463
+ begin
464
+ htpasswd = WEBrick::HTTPAuth::Htpasswd.new('dot.htpasswd')
465
+ htpasswd.set_passwd(nil, "API_KEY", "API_SECRET")
466
+ authenticator = WEBrick::HTTPAuth::BasicAuth.new(:UserDB => htpasswd, :Realm => "")
467
+ server.mount_proc('/') do |req, res|
468
+ authenticator.authenticate(req, res)
469
+ schema = File.read(File.join(__dir__, "..", "data", "schema-persions-value-1.avsc"))
470
+ res.body = schema
471
+ end
472
+ server.start
473
+ ensure
474
+ server.shutdown
475
+ end
476
+ end
477
+ end
478
+
479
+ def test_authentication_failure
480
+ conf = Fluent::Config::Element.new(
481
+ '', '', {'@type' => 'avro',
482
+ 'api_key' => 'WRONG_KEY',
483
+ 'api_secret' => 'WRONG_SECRET'
484
+ }, [
485
+ Fluent::Config::Element.new('confluent_registry', '', {
486
+ 'url' => 'http://localhost:8081',
487
+ 'subject' => 'persons-avro-value',
488
+ }, [])
489
+ ])
490
+ assert_raise(Fluent::Plugin::ConfluentAvroSchemaRegistryUnauthorizedError) do
491
+ d = create_driver(conf)
492
+ d.instance.run
493
+ end
494
+ end
495
+
496
+ def test_with_authentication
497
+ conf = Fluent::Config::Element.new(
498
+ '', '', {'@type' => 'avro',
499
+ 'api_key' => 'API_KEY',
500
+ 'api_secret' => 'API_SECRET'
501
+ }, [
502
+ Fluent::Config::Element.new('confluent_registry', '', {
503
+ 'url' => 'http://localhost:8081',
504
+ 'subject' => 'persons-avro-value',
505
+ }, [])
506
+ ])
507
+ datum = {"firstName" => "Aleen","lastName" => "Terry","birthDate" => 159202477258}
508
+ schema = Yajl.load(File.read(File.join(__dir__, "..", "data", "schema-persions-value-1.avsc")))
509
+ encoded = encode_datum(datum, schema.fetch("schema"), true, 1)
510
+ d = create_driver(conf)
511
+ d.instance.parse(encoded) do |_time, record|
512
+ assert_equal datum, record
513
+ end
514
+ end
515
+ end
516
+
453
517
  private
454
518
 
455
519
  def encode_datum(datum, string_schema, use_confluent_schema = true, schema_id = 1)
metadata CHANGED
@@ -1,14 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-parser-avro
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hiroshi Hatake
8
+ - Kentaro Hayashi
8
9
  autorequire:
9
10
  bindir: bin
10
11
  cert_chain: []
11
- date: 2020-09-30 00:00:00.000000000 Z
12
+ date: 2021-02-08 00:00:00.000000000 Z
12
13
  dependencies:
13
14
  - !ruby/object:Gem::Dependency
14
15
  name: avro
@@ -89,6 +90,7 @@ dependencies:
89
90
  description: Avro parser plugin for Fluentd
90
91
  email:
91
92
  - cosmo0920.wp@gmail.com
93
+ - kenhys@gmail.com
92
94
  executables: []
93
95
  extensions: []
94
96
  extra_rdoc_files: []