avro_turf 1.18.0 → 1.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +20 -11
- data/CHANGELOG.md +10 -0
- data/Gemfile +6 -1
- data/README.md +1 -1
- data/Rakefile +2 -1
- data/avro_turf.gemspec +17 -17
- data/lib/avro_turf/cached_confluent_schema_registry.rb +9 -8
- data/lib/avro_turf/cached_schema_registry.rb +3 -1
- data/lib/avro_turf/confluent_schema_registry.rb +35 -22
- data/lib/avro_turf/core_ext/date.rb +2 -0
- data/lib/avro_turf/core_ext/enumerable.rb +2 -0
- data/lib/avro_turf/core_ext/false_class.rb +2 -0
- data/lib/avro_turf/core_ext/hash.rb +4 -2
- data/lib/avro_turf/core_ext/nil_class.rb +2 -0
- data/lib/avro_turf/core_ext/numeric.rb +2 -0
- data/lib/avro_turf/core_ext/string.rb +2 -0
- data/lib/avro_turf/core_ext/symbol.rb +2 -0
- data/lib/avro_turf/core_ext/time.rb +2 -0
- data/lib/avro_turf/core_ext/true_class.rb +2 -0
- data/lib/avro_turf/core_ext.rb +12 -10
- data/lib/avro_turf/disk_cache.rb +13 -12
- data/lib/avro_turf/in_memory_cache.rb +2 -0
- data/lib/avro_turf/messaging.rb +31 -16
- data/lib/avro_turf/mutable_schema_store.rb +25 -4
- data/lib/avro_turf/schema_registry.rb +3 -1
- data/lib/avro_turf/schema_store.rb +3 -2
- data/lib/avro_turf/schema_to_avro_patch.rb +14 -12
- data/lib/avro_turf/test/fake_confluent_schema_registry_server.rb +24 -23
- data/lib/avro_turf/test/fake_prefixed_confluent_schema_registry_server.rb +12 -10
- data/lib/avro_turf/test/fake_schema_registry_server.rb +3 -1
- data/lib/avro_turf/version.rb +3 -1
- data/lib/avro_turf.rb +15 -13
- data/perf/encoding_size.rb +4 -2
- data/perf/encoding_speed.rb +4 -2
- data/spec/avro_turf_spec.rb +24 -23
- data/spec/cached_confluent_schema_registry_spec.rb +9 -7
- data/spec/confluent_schema_registry_spec.rb +31 -10
- data/spec/core_ext/date_spec.rb +2 -0
- data/spec/core_ext/enumerable_spec.rb +2 -0
- data/spec/core_ext/false_class_spec.rb +2 -0
- data/spec/core_ext/hash_spec.rb +3 -1
- data/spec/core_ext/nil_class_spec.rb +2 -0
- data/spec/core_ext/numeric_spec.rb +2 -0
- data/spec/core_ext/string_spec.rb +2 -0
- data/spec/core_ext/symbol_spec.rb +2 -0
- data/spec/core_ext/time_spec.rb +2 -0
- data/spec/core_ext/true_class_spec.rb +2 -0
- data/spec/disk_cached_confluent_schema_registry_spec.rb +23 -21
- data/spec/messaging_spec.rb +171 -86
- data/spec/mutable_schema_store_spec.rb +134 -0
- data/spec/schema_store_spec.rb +23 -21
- data/spec/schema_to_avro_patch_spec.rb +8 -7
- data/spec/spec_helper.rb +9 -9
- data/spec/support/authorized_fake_confluent_schema_registry_server.rb +4 -2
- data/spec/support/authorized_fake_prefixed_confluent_schema_registry_server.rb +4 -2
- data/spec/support/confluent_schema_registry_context.rb +32 -30
- data/spec/test/fake_confluent_schema_registry_server_spec.rb +97 -94
- metadata +15 -33
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 0f3d6b25c3e25a9137581fbe4bdb3e45015d371c479f8beeb54fa30ee8a3475e
|
|
4
|
+
data.tar.gz: a59d1634303784ae309a40c45f6abf70acdd5b681abe9a89bac40fa0497c5a63
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 85b800371ca6102518333c64732a26af7d103d35453923e1ac332b6c643e1826c00c90b826e9332c23e4de5f995bd40c8411447184289c2834fc45dbeded1664
|
|
7
|
+
data.tar.gz: df05accd07e4876a776a6dffb1850bfecd73d82727fc22ab5d25a4e002c069d50c7d9bb0e5fe9ae23e2f528b189c49d31ad87189d0ec07f5c2dcefb707110719
|
data/.github/workflows/ruby.yml
CHANGED
|
@@ -4,20 +4,29 @@ on: [push, pull_request]
|
|
|
4
4
|
|
|
5
5
|
jobs:
|
|
6
6
|
build:
|
|
7
|
-
|
|
8
7
|
runs-on: ubuntu-latest
|
|
9
8
|
strategy:
|
|
10
9
|
fail-fast: false
|
|
11
10
|
matrix:
|
|
12
|
-
ruby: [3.0, 3.1, 3.2, 3.3]
|
|
11
|
+
ruby: ["3.0", 3.1, 3.2, 3.3, 3.4]
|
|
12
|
+
steps:
|
|
13
|
+
- uses: actions/checkout@v5
|
|
14
|
+
- name: Set up Ruby ${{ matrix.ruby }}
|
|
15
|
+
uses: ruby/setup-ruby@v1
|
|
16
|
+
with:
|
|
17
|
+
ruby-version: ${{ matrix.ruby }}
|
|
18
|
+
bundler-cache: true
|
|
19
|
+
- name: Build and test with RSpec
|
|
20
|
+
env:
|
|
21
|
+
RUBYOPT: '--enable=frozen-string-literal --debug=frozen-string-literal'
|
|
22
|
+
run: bundle exec rspec
|
|
13
23
|
|
|
24
|
+
lint:
|
|
25
|
+
runs-on: ubuntu-24.04-arm
|
|
14
26
|
steps:
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
- name: Build and test with RSpec
|
|
22
|
-
run: |
|
|
23
|
-
bundle exec rspec
|
|
27
|
+
- uses: actions/checkout@v5
|
|
28
|
+
- uses: ruby/setup-ruby@v1
|
|
29
|
+
with:
|
|
30
|
+
ruby-version: 3
|
|
31
|
+
bundler-cache: true
|
|
32
|
+
- run: bundle exec standardrb
|
data/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,16 @@
|
|
|
2
2
|
|
|
3
3
|
## Unreleased
|
|
4
4
|
|
|
5
|
+
## v1.20.0
|
|
6
|
+
|
|
7
|
+
- Add support for client certificate chains via `client_chain` and `client_chain_data` parameters (#233)
|
|
8
|
+
- Ensure the gem works with frozen strings.
|
|
9
|
+
- Stop caching nested sub-schemas in MutableSchemaStore (#232)
|
|
10
|
+
|
|
11
|
+
## v1.19.0
|
|
12
|
+
|
|
13
|
+
- Loosen excon dependency to allow 1.x (#220)
|
|
14
|
+
|
|
5
15
|
## v1.18.0
|
|
6
16
|
|
|
7
17
|
- Add `compatibility_issues` method to `ConfluentSchemaRegistry` to debug compatibility issues between a schema versions for a given subject (#212)
|
data/Gemfile
CHANGED
data/README.md
CHANGED
|
@@ -282,7 +282,7 @@ require 'avro_turf/test/fake_confluent_schema_registry_server'
|
|
|
282
282
|
require 'webmock/rspec'
|
|
283
283
|
|
|
284
284
|
class AuthorizedFakeConfluentSchemaRegistryServer < FakeConfluentSchemaRegistryServer
|
|
285
|
-
set :
|
|
285
|
+
set :host_authorization, permitted_hosts: ['registry.example.com']
|
|
286
286
|
end
|
|
287
287
|
|
|
288
288
|
# within an example
|
data/Rakefile
CHANGED
data/avro_turf.gemspec
CHANGED
|
@@ -1,38 +1,38 @@
|
|
|
1
|
-
#
|
|
2
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
lib = File.expand_path("../lib", __FILE__)
|
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
|
-
require
|
|
5
|
+
require "avro_turf/version"
|
|
5
6
|
|
|
6
7
|
Gem::Specification.new do |spec|
|
|
7
|
-
spec.name
|
|
8
|
-
spec.version
|
|
9
|
-
spec.authors
|
|
10
|
-
spec.email
|
|
11
|
-
spec.summary
|
|
12
|
-
spec.homepage
|
|
13
|
-
spec.license
|
|
8
|
+
spec.name = "avro_turf"
|
|
9
|
+
spec.version = AvroTurf::VERSION
|
|
10
|
+
spec.authors = ["Daniel Schierbeck"]
|
|
11
|
+
spec.email = ["dasch@zendesk.com"]
|
|
12
|
+
spec.summary = "A library that makes it easier to use the Avro serialization format from Ruby"
|
|
13
|
+
spec.homepage = "https://github.com/dasch/avro_turf"
|
|
14
|
+
spec.license = "MIT"
|
|
14
15
|
|
|
15
16
|
spec.metadata["rubygems_mfa_required"] = "true"
|
|
16
17
|
|
|
17
|
-
spec.files
|
|
18
|
-
spec.executables
|
|
19
|
-
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
|
18
|
+
spec.files = `git ls-files -z`.split("\x0")
|
|
19
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
|
20
20
|
spec.require_paths = ["lib"]
|
|
21
21
|
|
|
22
22
|
spec.add_dependency "avro", ">= 1.11.3", "< 1.13"
|
|
23
|
-
spec.add_dependency "excon", "
|
|
23
|
+
spec.add_dependency "excon", ">= 0.104", "< 2"
|
|
24
24
|
|
|
25
25
|
spec.add_development_dependency "bundler", "~> 2.0"
|
|
26
26
|
spec.add_development_dependency "rake", "~> 13.0"
|
|
27
27
|
spec.add_development_dependency "rspec", "~> 3.2"
|
|
28
|
-
spec.add_development_dependency "fakefs", "
|
|
28
|
+
spec.add_development_dependency "fakefs", "~> 3"
|
|
29
29
|
spec.add_development_dependency "webmock"
|
|
30
30
|
spec.add_development_dependency "sinatra"
|
|
31
31
|
spec.add_development_dependency "json_spec"
|
|
32
32
|
spec.add_development_dependency "rack-test"
|
|
33
33
|
spec.add_development_dependency "resolv"
|
|
34
34
|
|
|
35
|
-
spec.post_install_message = %
|
|
35
|
+
spec.post_install_message = %(
|
|
36
36
|
avro_turf v0.8.0 deprecates the names AvroTurf::SchemaRegistry,
|
|
37
37
|
AvroTurf::CachedSchemaRegistry, and FakeSchemaRegistryServer.
|
|
38
38
|
|
|
@@ -40,5 +40,5 @@ Use AvroTurf::ConfluentSchemaRegistry, AvroTurf::CachedConfluentSchemaRegistry,
|
|
|
40
40
|
and FakeConfluentSchemaRegistryServer instead.
|
|
41
41
|
|
|
42
42
|
See https://github.com/dasch/avro_turf#deprecation-notice
|
|
43
|
-
|
|
43
|
+
)
|
|
44
44
|
end
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
require
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "avro_turf/confluent_schema_registry"
|
|
4
|
+
require "avro_turf/in_memory_cache"
|
|
5
|
+
require "avro_turf/disk_cache"
|
|
4
6
|
|
|
5
7
|
# Caches registrations and lookups to the schema registry in memory.
|
|
6
8
|
class AvroTurf::CachedConfluentSchemaRegistry
|
|
7
|
-
|
|
8
9
|
# Instantiate a new CachedConfluentSchemaRegistry instance with the given configuration.
|
|
9
10
|
# By default, uses a provided InMemoryCache to prevent repeated calls to the upstream registry.
|
|
10
11
|
#
|
|
@@ -17,8 +18,8 @@ class AvroTurf::CachedConfluentSchemaRegistry
|
|
|
17
18
|
end
|
|
18
19
|
|
|
19
20
|
# Delegate the following methods to the upstream
|
|
20
|
-
%i
|
|
21
|
-
|
|
21
|
+
%i[subjects subject_versions schema_subject_versions compatible?
|
|
22
|
+
global_config update_global_config subject_config update_subject_config].each do |name|
|
|
22
23
|
define_method(name) do |*args|
|
|
23
24
|
instance_variable_get(:@upstream).send(name, *args)
|
|
24
25
|
end
|
|
@@ -36,8 +37,8 @@ class AvroTurf::CachedConfluentSchemaRegistry
|
|
|
36
37
|
@cache.lookup_by_schema(subject, schema) || @cache.store_by_schema(subject, schema, @upstream.register(subject, schema))
|
|
37
38
|
end
|
|
38
39
|
|
|
39
|
-
def subject_version(subject, version =
|
|
40
|
-
return @upstream.subject_version(subject, version) if version ==
|
|
40
|
+
def subject_version(subject, version = "latest")
|
|
41
|
+
return @upstream.subject_version(subject, version) if version == "latest"
|
|
41
42
|
|
|
42
43
|
@cache.lookup_by_version(subject, version) ||
|
|
43
44
|
@cache.store_by_version(subject, version, @upstream.subject_version(subject, version))
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "avro_turf/cached_confluent_schema_registry"
|
|
2
4
|
|
|
3
5
|
# AvroTurf::CachedSchemaRegistry is deprecated and will be removed in a future release.
|
|
4
6
|
# Use AvroTurf::CachedConfluentSchemaRegistry instead.
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "excon"
|
|
2
4
|
|
|
3
5
|
class AvroTurf::ConfluentSchemaRegistry
|
|
4
|
-
CONTENT_TYPE = "application/vnd.schemaregistry.v1+json"
|
|
6
|
+
CONTENT_TYPE = "application/vnd.schemaregistry.v1+json"
|
|
5
7
|
|
|
6
8
|
def initialize(
|
|
7
9
|
url,
|
|
@@ -12,46 +14,57 @@ class AvroTurf::ConfluentSchemaRegistry
|
|
|
12
14
|
password: nil,
|
|
13
15
|
ssl_ca_file: nil,
|
|
14
16
|
client_cert: nil,
|
|
17
|
+
client_chain: nil,
|
|
15
18
|
client_key: nil,
|
|
16
19
|
client_key_pass: nil,
|
|
17
20
|
client_cert_data: nil,
|
|
21
|
+
client_chain_data: nil,
|
|
18
22
|
client_key_data: nil,
|
|
19
23
|
path_prefix: nil,
|
|
20
24
|
connect_timeout: nil,
|
|
21
|
-
resolv_resolver: nil
|
|
25
|
+
resolv_resolver: nil,
|
|
26
|
+
retry_limit: nil
|
|
22
27
|
)
|
|
23
28
|
@path_prefix = path_prefix
|
|
24
|
-
@schema_context_prefix = schema_context.nil? ?
|
|
29
|
+
@schema_context_prefix = schema_context.nil? ? "" : ":.#{schema_context}:"
|
|
25
30
|
@schema_context_options = schema_context.nil? ? {} : {query: {subject: @schema_context_prefix}}
|
|
26
31
|
@logger = logger
|
|
27
32
|
headers = Excon.defaults[:headers].merge(
|
|
28
33
|
"Content-Type" => CONTENT_TYPE
|
|
29
34
|
)
|
|
30
|
-
|
|
31
|
-
@connection = Excon.new(
|
|
32
|
-
url,
|
|
35
|
+
params = {
|
|
33
36
|
headers: headers,
|
|
34
37
|
user: user,
|
|
35
38
|
password: password,
|
|
39
|
+
proxy: proxy,
|
|
36
40
|
ssl_ca_file: ssl_ca_file,
|
|
37
41
|
client_cert: client_cert,
|
|
42
|
+
client_chain: client_chain,
|
|
38
43
|
client_key: client_key,
|
|
39
44
|
client_key_pass: client_key_pass,
|
|
40
45
|
client_cert_data: client_cert_data,
|
|
46
|
+
client_chain_data: client_chain_data,
|
|
41
47
|
client_key_data: client_key_data,
|
|
48
|
+
resolv_resolver: resolv_resolver,
|
|
42
49
|
connect_timeout: connect_timeout,
|
|
43
|
-
|
|
50
|
+
retry_limit: retry_limit
|
|
51
|
+
}
|
|
52
|
+
# Remove nil params to allow Excon to use its default values
|
|
53
|
+
params.reject! { |_, v| v.nil? }
|
|
54
|
+
@connection = Excon.new(
|
|
55
|
+
url,
|
|
56
|
+
params
|
|
44
57
|
)
|
|
45
58
|
end
|
|
46
59
|
|
|
47
60
|
def fetch(id)
|
|
48
61
|
@logger.info "Fetching schema with id #{id}"
|
|
49
|
-
data = get("/schemas/ids/#{id}", idempotent: true, **@schema_context_options
|
|
62
|
+
data = get("/schemas/ids/#{id}", idempotent: true, **@schema_context_options)
|
|
50
63
|
data.fetch("schema")
|
|
51
64
|
end
|
|
52
65
|
|
|
53
66
|
def register(subject, schema)
|
|
54
|
-
data = post("/subjects/#{@schema_context_prefix}#{subject}/versions", body: {
|
|
67
|
+
data = post("/subjects/#{@schema_context_prefix}#{subject}/versions", body: {schema: schema.to_s}.to_json)
|
|
55
68
|
|
|
56
69
|
id = data.fetch("id")
|
|
57
70
|
|
|
@@ -62,7 +75,7 @@ class AvroTurf::ConfluentSchemaRegistry
|
|
|
62
75
|
|
|
63
76
|
# List all subjects
|
|
64
77
|
def subjects
|
|
65
|
-
get(
|
|
78
|
+
get("/subjects", idempotent: true)
|
|
66
79
|
end
|
|
67
80
|
|
|
68
81
|
# List all versions for a subject
|
|
@@ -71,7 +84,7 @@ class AvroTurf::ConfluentSchemaRegistry
|
|
|
71
84
|
end
|
|
72
85
|
|
|
73
86
|
# Get a specific version for a subject
|
|
74
|
-
def subject_version(subject, version =
|
|
87
|
+
def subject_version(subject, version = "latest")
|
|
75
88
|
get("/subjects/#{@schema_context_prefix}#{subject}/versions/#{version}", idempotent: true)
|
|
76
89
|
end
|
|
77
90
|
|
|
@@ -83,9 +96,9 @@ class AvroTurf::ConfluentSchemaRegistry
|
|
|
83
96
|
# Check if a schema exists. Returns nil if not found.
|
|
84
97
|
def check(subject, schema)
|
|
85
98
|
data = post("/subjects/#{@schema_context_prefix}#{subject}",
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
99
|
+
expects: [200, 404],
|
|
100
|
+
body: {schema: schema.to_s}.to_json,
|
|
101
|
+
idempotent: true)
|
|
89
102
|
data unless data.has_key?("error_code")
|
|
90
103
|
end
|
|
91
104
|
|
|
@@ -95,10 +108,10 @@ class AvroTurf::ConfluentSchemaRegistry
|
|
|
95
108
|
# - nil if the subject or version does not exist
|
|
96
109
|
# - false if incompatible
|
|
97
110
|
# http://docs.confluent.io/3.1.2/schema-registry/docs/api.html#compatibility
|
|
98
|
-
def compatible?(subject, schema, version =
|
|
111
|
+
def compatible?(subject, schema, version = "latest")
|
|
99
112
|
data = post("/compatibility/subjects/#{@schema_context_prefix}#{subject}/versions/#{version}",
|
|
100
|
-
|
|
101
|
-
data.fetch(
|
|
113
|
+
expects: [200, 404], body: {schema: schema.to_s}.to_json, idempotent: true)
|
|
114
|
+
data.fetch("is_compatible", false) unless data.has_key?("error_code")
|
|
102
115
|
end
|
|
103
116
|
|
|
104
117
|
# Check for specific schema compatibility issues
|
|
@@ -106,11 +119,11 @@ class AvroTurf::ConfluentSchemaRegistry
|
|
|
106
119
|
# - nil if the subject or version does not exist
|
|
107
120
|
# - a list of compatibility issues
|
|
108
121
|
# https://docs.confluent.io/platform/current/schema-registry/develop/api.html#sr-api-compatibility
|
|
109
|
-
def compatibility_issues(subject, schema, version =
|
|
122
|
+
def compatibility_issues(subject, schema, version = "latest")
|
|
110
123
|
data = post("/compatibility/subjects/#{@schema_context_prefix}#{subject}/versions/#{version}",
|
|
111
|
-
expects: [200, 404], body: {
|
|
124
|
+
expects: [200, 404], body: {schema: schema.to_s}.to_json, query: {verbose: true}, idempotent: true)
|
|
112
125
|
|
|
113
|
-
data.fetch(
|
|
126
|
+
data.fetch("messages", []) unless data.has_key?("error_code")
|
|
114
127
|
end
|
|
115
128
|
|
|
116
129
|
# Get global config
|
|
@@ -148,7 +161,7 @@ class AvroTurf::ConfluentSchemaRegistry
|
|
|
148
161
|
end
|
|
149
162
|
|
|
150
163
|
def request(path, **options)
|
|
151
|
-
options = {
|
|
164
|
+
options = {expects: 200}.merge!(options)
|
|
152
165
|
path = File.join(@path_prefix, path) unless @path_prefix.nil?
|
|
153
166
|
response = @connection.request(path: path, **options)
|
|
154
167
|
JSON.parse(response.body)
|
data/lib/avro_turf/core_ext.rb
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
require
|
|
4
|
-
require
|
|
5
|
-
require
|
|
6
|
-
require
|
|
7
|
-
require
|
|
8
|
-
require
|
|
9
|
-
require
|
|
10
|
-
require
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "avro_turf/core_ext/string"
|
|
4
|
+
require "avro_turf/core_ext/numeric"
|
|
5
|
+
require "avro_turf/core_ext/enumerable"
|
|
6
|
+
require "avro_turf/core_ext/hash"
|
|
7
|
+
require "avro_turf/core_ext/time"
|
|
8
|
+
require "avro_turf/core_ext/date"
|
|
9
|
+
require "avro_turf/core_ext/symbol"
|
|
10
|
+
require "avro_turf/core_ext/nil_class"
|
|
11
|
+
require "avro_turf/core_ext/true_class"
|
|
12
|
+
require "avro_turf/core_ext/false_class"
|
data/lib/avro_turf/disk_cache.rb
CHANGED
|
@@ -1,23 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
# A cache for the CachedConfluentSchemaRegistry.
|
|
2
4
|
# Extends the InMemoryCache to provide a write-thru to disk for persistent cache.
|
|
3
5
|
class AvroTurf::DiskCache
|
|
4
|
-
|
|
5
6
|
def initialize(disk_path, logger: Logger.new($stdout))
|
|
6
7
|
@logger = logger
|
|
7
8
|
|
|
8
9
|
# load the write-thru cache on startup, if it exists
|
|
9
|
-
@schemas_by_id_path = File.join(disk_path,
|
|
10
|
+
@schemas_by_id_path = File.join(disk_path, "schemas_by_id.json")
|
|
10
11
|
hash = read_from_disk_cache(@schemas_by_id_path)
|
|
11
12
|
@schemas_by_id = hash || {}
|
|
12
13
|
|
|
13
|
-
@ids_by_schema_path = File.join(disk_path,
|
|
14
|
+
@ids_by_schema_path = File.join(disk_path, "ids_by_schema.json")
|
|
14
15
|
hash = read_from_disk_cache(@ids_by_schema_path)
|
|
15
16
|
@ids_by_schema = hash || {}
|
|
16
17
|
|
|
17
|
-
@schemas_by_subject_version_path = File.join(disk_path,
|
|
18
|
+
@schemas_by_subject_version_path = File.join(disk_path, "schemas_by_subject_version.json")
|
|
18
19
|
@schemas_by_subject_version = {}
|
|
19
20
|
|
|
20
|
-
@data_by_schema_path = File.join(disk_path,
|
|
21
|
+
@data_by_schema_path = File.join(disk_path, "data_by_schema.json")
|
|
21
22
|
hash = read_from_disk_cache(@data_by_schema_path)
|
|
22
23
|
@data_by_schema = hash || {}
|
|
23
24
|
end
|
|
@@ -94,11 +95,11 @@ class AvroTurf::DiskCache
|
|
|
94
95
|
key = "#{subject}#{version}"
|
|
95
96
|
hash = read_from_disk_cache(@schemas_by_subject_version_path)
|
|
96
97
|
hash = if hash
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
98
|
+
hash[key] = schema
|
|
99
|
+
hash
|
|
100
|
+
else
|
|
101
|
+
{key => schema}
|
|
102
|
+
end
|
|
102
103
|
|
|
103
104
|
write_to_disk_cache(@schemas_by_subject_version_path, hash)
|
|
104
105
|
|
|
@@ -110,7 +111,7 @@ class AvroTurf::DiskCache
|
|
|
110
111
|
private def read_from_disk_cache(path)
|
|
111
112
|
if File.exist?(path)
|
|
112
113
|
if File.size(path) != 0
|
|
113
|
-
json_data = File.open(path,
|
|
114
|
+
json_data = File.open(path, "r") do |file|
|
|
114
115
|
file.flock(File::LOCK_SH)
|
|
115
116
|
file.read
|
|
116
117
|
end
|
|
@@ -127,7 +128,7 @@ class AvroTurf::DiskCache
|
|
|
127
128
|
|
|
128
129
|
private def write_to_disk_cache(path, hash)
|
|
129
130
|
# don't use "w" because it truncates the file before lock
|
|
130
|
-
File.open(path, File::RDWR | File::CREAT,
|
|
131
|
+
File.open(path, File::RDWR | File::CREAT, 0o644) do |file|
|
|
131
132
|
file.flock(File::LOCK_EX)
|
|
132
133
|
file.write(JSON.pretty_generate(hash))
|
|
133
134
|
end
|
data/lib/avro_turf/messaging.rb
CHANGED
|
@@ -1,15 +1,18 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
require
|
|
4
|
-
require
|
|
5
|
-
require
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "logger"
|
|
4
|
+
require "avro_turf"
|
|
5
|
+
require "avro_turf/schema_store"
|
|
6
|
+
require "avro_turf/confluent_schema_registry"
|
|
7
|
+
require "avro_turf/cached_confluent_schema_registry"
|
|
6
8
|
|
|
7
9
|
# For back-compatibility require the aliases along with the Messaging API.
|
|
8
10
|
# These names are deprecated and will be removed in a future release.
|
|
9
|
-
require
|
|
10
|
-
require
|
|
11
|
+
require "avro_turf/schema_registry"
|
|
12
|
+
require "avro_turf/cached_schema_registry"
|
|
11
13
|
|
|
12
14
|
class AvroTurf
|
|
15
|
+
class IncompatibleSchemaError < StandardError; end
|
|
13
16
|
|
|
14
17
|
# Provides a way to encode and decode messages without having to embed schemas
|
|
15
18
|
# in the encoded data. Confluent's Schema Registry[1] is used to register
|
|
@@ -55,7 +58,9 @@ class AvroTurf
|
|
|
55
58
|
# client_key - Name of file containing client private key to go with client_cert (optional).
|
|
56
59
|
# client_key_pass - Password to go with client_key (optional).
|
|
57
60
|
# client_cert_data - In-memory client certificate (optional).
|
|
61
|
+
# client_chain - Name of file containing client certificate chain (optional).
|
|
58
62
|
# client_key_data - In-memory client private key to go with client_cert_data (optional).
|
|
63
|
+
# client_chain_data - In-memory client certificate chain (optional).
|
|
59
64
|
# connect_timeout - Timeout to use in the connection with the schema registry (optional).
|
|
60
65
|
# resolv_resolver - Custom domain name resolver (optional).
|
|
61
66
|
def initialize(
|
|
@@ -72,12 +77,15 @@ class AvroTurf
|
|
|
72
77
|
password: nil,
|
|
73
78
|
ssl_ca_file: nil,
|
|
74
79
|
client_cert: nil,
|
|
80
|
+
client_chain: nil,
|
|
75
81
|
client_key: nil,
|
|
76
82
|
client_key_pass: nil,
|
|
77
83
|
client_cert_data: nil,
|
|
84
|
+
client_chain_data: nil,
|
|
78
85
|
client_key_data: nil,
|
|
79
86
|
connect_timeout: nil,
|
|
80
|
-
resolv_resolver: nil
|
|
87
|
+
resolv_resolver: nil,
|
|
88
|
+
retry_limit: nil
|
|
81
89
|
)
|
|
82
90
|
@logger = logger || Logger.new($stderr)
|
|
83
91
|
@namespace = namespace
|
|
@@ -92,13 +100,16 @@ class AvroTurf
|
|
|
92
100
|
password: password,
|
|
93
101
|
ssl_ca_file: ssl_ca_file,
|
|
94
102
|
client_cert: client_cert,
|
|
103
|
+
client_chain: client_chain,
|
|
95
104
|
client_key: client_key,
|
|
96
105
|
client_key_pass: client_key_pass,
|
|
97
106
|
client_cert_data: client_cert_data,
|
|
107
|
+
client_chain_data: client_chain_data,
|
|
98
108
|
client_key_data: client_key_data,
|
|
99
109
|
path_prefix: registry_path_prefix,
|
|
100
110
|
connect_timeout: connect_timeout,
|
|
101
|
-
resolv_resolver: resolv_resolver
|
|
111
|
+
resolv_resolver: resolv_resolver,
|
|
112
|
+
retry_limit: retry_limit
|
|
102
113
|
)
|
|
103
114
|
)
|
|
104
115
|
@schemas_by_id = {}
|
|
@@ -126,7 +137,7 @@ class AvroTurf
|
|
|
126
137
|
#
|
|
127
138
|
# Returns the encoded data as a String.
|
|
128
139
|
def encode(message, schema_name: nil, namespace: @namespace, subject: nil, version: nil, schema_id: nil, validate: false,
|
|
129
|
-
|
|
140
|
+
register_schemas: true)
|
|
130
141
|
schema, schema_id = if schema_id
|
|
131
142
|
fetch_schema_by_id(schema_id)
|
|
132
143
|
elsif subject && version
|
|
@@ -136,7 +147,7 @@ class AvroTurf
|
|
|
136
147
|
elsif schema_name
|
|
137
148
|
register_schema(subject: subject, schema_name: schema_name, namespace: namespace)
|
|
138
149
|
else
|
|
139
|
-
raise ArgumentError.new(
|
|
150
|
+
raise ArgumentError.new("Neither schema_name nor schema_id nor subject + version provided to determine the schema.")
|
|
140
151
|
end
|
|
141
152
|
|
|
142
153
|
if validate
|
|
@@ -200,7 +211,7 @@ class AvroTurf
|
|
|
200
211
|
end
|
|
201
212
|
|
|
202
213
|
# The schema id is a 4-byte big-endian integer.
|
|
203
|
-
schema_id = decoder.read(4).
|
|
214
|
+
schema_id = decoder.read(4).unpack1("N")
|
|
204
215
|
|
|
205
216
|
writers_schema = @schemas_by_id.fetch(schema_id) do
|
|
206
217
|
schema_json = @registry.fetch(schema_id)
|
|
@@ -216,12 +227,16 @@ class AvroTurf
|
|
|
216
227
|
end
|
|
217
228
|
|
|
218
229
|
# Providing subject and version to determine the schema,
|
|
219
|
-
# which skips the auto
|
|
230
|
+
# which skips the auto registration of schema on the schema registry.
|
|
220
231
|
# Fetch the schema from registry with the provided subject name and version.
|
|
221
|
-
def fetch_schema(subject:, version:
|
|
232
|
+
def fetch_schema(subject:, version: "latest")
|
|
222
233
|
schema_data = @registry.subject_version(subject, version)
|
|
223
|
-
schema_id = schema_data.fetch(
|
|
224
|
-
|
|
234
|
+
schema_id = schema_data.fetch("id")
|
|
235
|
+
schema_type = schema_data["schemaType"]
|
|
236
|
+
if schema_type && schema_type != "AVRO"
|
|
237
|
+
raise IncompatibleSchemaError, "The #{schema_type} schema for #{subject} is incompatible."
|
|
238
|
+
end
|
|
239
|
+
schema = Avro::Schema.parse(schema_data.fetch("schema"))
|
|
225
240
|
[schema, schema_id]
|
|
226
241
|
end
|
|
227
242
|
|