karafka-avro-parser 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +14 -0
- data/.rspec +2 -0
- data/.travis.yml +5 -0
- data/Gemfile +6 -0
- data/README.md +77 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/karafka-avro-parser.gemspec +26 -0
- data/lib/karafka/parsers/avro/parser.rb +25 -0
- data/lib/karafka/parsers/avro/version.rb +7 -0
- data/lib/karafka/parsers/avro.rb +31 -0
- data/lib/karafka-avro-parser.rb +1 -0
- metadata +112 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 0851cb76a5deab8dc8365afcb5a4f2e83b9a62a1
|
4
|
+
data.tar.gz: 61bd7f4d7fda7ac0cb5d9d250abd04da8e2f63da
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: d5b119b00217c76a584b748cc145cab0b5c6d33e5e1ec6c739daeb271f7f3311189677c926a8f4f764b11458f36d4537a61302e4906ba615bbae06457f53a9da
|
7
|
+
data.tar.gz: 4e0535ebc08886e460be4b32cb0c4a95e92681357a167121921d628e3fb6ac7b99a36646b84a4a087f016f6bdf21fbfbc6d45f1873151eb351694de603b76b65
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,77 @@
|
|
1
|
+
# Karafka::Avro::Parser
|
2
|
+
|
3
|
+
Karafka Parser for [Apache Avro](http://avro.apache.org/). It uses the great [AvroTurf](https://github.com/dasch/avro_turf) gem by under the hood.
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
gem 'karafka-avro-parser'
|
11
|
+
```
|
12
|
+
|
13
|
+
And then execute:
|
14
|
+
|
15
|
+
$ bundle
|
16
|
+
|
17
|
+
## Usage
|
18
|
+
|
19
|
+
Thanks to Karafka's modular design, you only have to define a `parser` the moment you draw your consumer groups.
|
20
|
+
|
21
|
+
### Example with Schema Folder
|
22
|
+
|
23
|
+
```ruby
|
24
|
+
Karafka::Parsers::Avro.schemas_path = 'app/schemas'
|
25
|
+
|
26
|
+
App.consumer_groups.draw do
|
27
|
+
consumer_group :my_consumer_group do
|
28
|
+
topic :my_topic do
|
29
|
+
consumer AvroConsumer
|
30
|
+
parser Karafka::Parsers::Avro.from_path('schema_name')
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
```
|
35
|
+
|
36
|
+
### Example with a Codec
|
37
|
+
|
38
|
+
```ruby
|
39
|
+
Karafka::Parsers::Avro.schemas_path = 'app/schemas'
|
40
|
+
|
41
|
+
App.consumer_groups.draw do
|
42
|
+
consumer_group :my_consumer_group do
|
43
|
+
topic :my_topic do
|
44
|
+
consumer AvroConsumer
|
45
|
+
parser Karafka::Parsers::Avro.from_path('schema_name', codec: 'deflate')
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
```
|
50
|
+
|
51
|
+
### Example with Schema Registry
|
52
|
+
|
53
|
+
```ruby
|
54
|
+
Karafka::Parsers::Avro.registry_url = 'http://my-registry:8081/'
|
55
|
+
|
56
|
+
App.consumer_groups.draw do
|
57
|
+
consumer_group :my_consumer_group do
|
58
|
+
topic :my_topic do
|
59
|
+
consumer AvroConsumer
|
60
|
+
parser Karafka::Parsers::Avro.from_registry('schema_name')
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
```
|
65
|
+
|
66
|
+
## Note on contributions
|
67
|
+
|
68
|
+
First, thank you for considering contributing to Karafka! It's people like you that make the open source community such a great community!
|
69
|
+
|
70
|
+
Each pull request must pass all the RSpec specs and meet our quality requirements.
|
71
|
+
|
72
|
+
To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combines multiple linters and code analyzers for both code and documentation. Once you're done with your changes, submit a pull request.
|
73
|
+
|
74
|
+
Coditsu will automatically check your work against our quality standards. You can find your commit check results on the [builds page](https://app.coditsu.io/karafka/commit_builds) of Karafka organization.
|
75
|
+
|
76
|
+
[![coditsu](https://coditsu.io/assets/quality_bar.svg)](https://app.coditsu.io/karafka-avro/commit_builds)
|
77
|
+
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "karafka/avro/parser"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path("../lib", __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'karafka/parsers/avro/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = 'karafka-avro-parser'
|
8
|
+
spec.version = Karafka::Parsers::Avro::VERSION
|
9
|
+
spec.authors = ['Kacper Madej']
|
10
|
+
spec.email = %w[kacperoza@gmail.com]
|
11
|
+
|
12
|
+
spec.summary = %q{Apache Avro support for Karafka}
|
13
|
+
spec.description = %q{}
|
14
|
+
spec.homepage = 'https://github.com/karafka/avro'
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
17
|
+
f.match(%r{^(test|spec|features)/})
|
18
|
+
end
|
19
|
+
spec.require_paths = %w[lib]
|
20
|
+
|
21
|
+
spec.add_dependency 'avro_turf', '~> 0.8'
|
22
|
+
|
23
|
+
spec.add_development_dependency 'bundler', '~> 1.15'
|
24
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
25
|
+
spec.add_development_dependency 'rspec', '~> 3.0'
|
26
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'avro_turf'
|
2
|
+
require 'avro_turf/messaging'
|
3
|
+
|
4
|
+
module Karafka
|
5
|
+
module Parsers
|
6
|
+
module Avro
|
7
|
+
class Parser
|
8
|
+
attr_reader :avro, :schema_name
|
9
|
+
|
10
|
+
def initialize(avro, schema_name)
|
11
|
+
@avro = avro
|
12
|
+
@schema_name = schema_name
|
13
|
+
end
|
14
|
+
|
15
|
+
def parse(content)
|
16
|
+
avro.decode(content, schema_name: schema_name)
|
17
|
+
end
|
18
|
+
|
19
|
+
def generate(content)
|
20
|
+
avro.encode(content, schema_name: schema_name)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'avro_turf'
|
2
|
+
require 'avro_turf/messaging'
|
3
|
+
|
4
|
+
require_relative 'avro/parser'
|
5
|
+
require_relative 'avro/version'
|
6
|
+
|
7
|
+
module Karafka
|
8
|
+
module Parsers
|
9
|
+
module Avro
|
10
|
+
def self.registry_url=(registry_url)
|
11
|
+
@registry_url = registry_url
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.schemas_path=(schemas_path)
|
15
|
+
@schemas_path = schemas_path
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.from_registry(schema_name)
|
19
|
+
raise ArgumentError, 'You have to specify registry_path first' if @registry_url.nil?
|
20
|
+
|
21
|
+
Parser.new(AvroTurf::Messaging.new(registry_url: @registry_url), schema_name)
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.from_path(schema_name, codec: nil)
|
25
|
+
raise ArgumentError, 'You have to specify schemas_path first' if @schemas_path.nil?
|
26
|
+
|
27
|
+
Parser.new(AvroTurf.new(schemas_path: @schemas_path, codec: codec), schema_name)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
require_relative 'karafka/parsers/avro'
|
metadata
ADDED
@@ -0,0 +1,112 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: karafka-avro-parser
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Kacper Madej
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2018-04-03 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: avro_turf
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0.8'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0.8'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bundler
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.15'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.15'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '10.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '10.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rspec
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '3.0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '3.0'
|
69
|
+
description: ''
|
70
|
+
email:
|
71
|
+
- kacperoza@gmail.com
|
72
|
+
executables: []
|
73
|
+
extensions: []
|
74
|
+
extra_rdoc_files: []
|
75
|
+
files:
|
76
|
+
- ".gitignore"
|
77
|
+
- ".rspec"
|
78
|
+
- ".travis.yml"
|
79
|
+
- Gemfile
|
80
|
+
- README.md
|
81
|
+
- Rakefile
|
82
|
+
- bin/console
|
83
|
+
- bin/setup
|
84
|
+
- karafka-avro-parser.gemspec
|
85
|
+
- lib/karafka-avro-parser.rb
|
86
|
+
- lib/karafka/parsers/avro.rb
|
87
|
+
- lib/karafka/parsers/avro/parser.rb
|
88
|
+
- lib/karafka/parsers/avro/version.rb
|
89
|
+
homepage: https://github.com/karafka/avro
|
90
|
+
licenses: []
|
91
|
+
metadata: {}
|
92
|
+
post_install_message:
|
93
|
+
rdoc_options: []
|
94
|
+
require_paths:
|
95
|
+
- lib
|
96
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
97
|
+
requirements:
|
98
|
+
- - ">="
|
99
|
+
- !ruby/object:Gem::Version
|
100
|
+
version: '0'
|
101
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
102
|
+
requirements:
|
103
|
+
- - ">="
|
104
|
+
- !ruby/object:Gem::Version
|
105
|
+
version: '0'
|
106
|
+
requirements: []
|
107
|
+
rubyforge_project:
|
108
|
+
rubygems_version: 2.5.1
|
109
|
+
signing_key:
|
110
|
+
specification_version: 4
|
111
|
+
summary: Apache Avro support for Karafka
|
112
|
+
test_files: []
|