qian 0.0.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +53 -0
- data/.gitmodules +3 -0
- data/.rspec +1 -0
- data/CHANGLOG.md +0 -0
- data/Gemfile +3 -0
- data/README.md +1 -0
- data/Rakefile +9 -0
- data/lib/qian.rb +123 -0
- data/lib/qian/event.rb +71 -0
- data/lib/qian/events/engagement.rb +23 -0
- data/lib/qian/logger.rb +39 -0
- data/lib/qian/util.rb +18 -0
- data/lib/qian/version.rb +3 -0
- data/qian.gemspec +32 -0
- data/spec/qian/event_spec.rb +113 -0
- data/spec/qian/logger_spec.rb +49 -0
- data/spec/qian/util_spec.rb +9 -0
- data/spec/qian_spec.rb +62 -0
- data/spec/spec_helper.rb +18 -0
- data/spec/support/avro_schema/com/jianshu/event/test/Engagement.avsc +58 -0
- metadata +196 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 4cad90744ce3d8c2829f6d0392532da15d07a8cf
|
4
|
+
data.tar.gz: 4ab48952d8d657516f32552552f30cccff767e33
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 27cc97ad7d7c0b2c6ebe15122f2f98260884a9058696d45aea283a213537a76e6d12cf8f01a52a64dda8332cd2247fcb708e29ef004da4794b1379630e9d5361
|
7
|
+
data.tar.gz: 274af45bc42f8d8a3409c02229211ec9e34d0b235cd04ce0f1b1ec6ffe175fac36899365a00d71a38b6b36c674fb5bd5f954523a1a672249822eb9e52e2c1398
|
data/.gitignore
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
.DS_Store
|
2
|
+
*.gem
|
3
|
+
*.rbc
|
4
|
+
.bundle
|
5
|
+
.ruby-gemset
|
6
|
+
.ruby-version
|
7
|
+
.config
|
8
|
+
.yardoc
|
9
|
+
Gemfile.lock
|
10
|
+
InstalledFiles
|
11
|
+
_yardoc
|
12
|
+
coverage
|
13
|
+
doc/
|
14
|
+
lib/bundler/man
|
15
|
+
pkg
|
16
|
+
rdoc
|
17
|
+
spec/reports
|
18
|
+
test/tmp
|
19
|
+
test/version_tmp
|
20
|
+
tmp
|
21
|
+
/.config
|
22
|
+
/coverage/
|
23
|
+
/InstalledFiles
|
24
|
+
/pkg/
|
25
|
+
/spec/reports/
|
26
|
+
/test/tmp/
|
27
|
+
/test/version_tmp/
|
28
|
+
/tmp/
|
29
|
+
/gemfiles/*.lock
|
30
|
+
|
31
|
+
## Specific to RubyMotion:
|
32
|
+
.dat*
|
33
|
+
.repl_history
|
34
|
+
build/
|
35
|
+
|
36
|
+
## Documentation cache and generated files:
|
37
|
+
/.yardoc/
|
38
|
+
/_yardoc/
|
39
|
+
/doc/
|
40
|
+
/rdoc/
|
41
|
+
|
42
|
+
## Environment normalisation:
|
43
|
+
/.bundle/
|
44
|
+
/lib/bundler/man/
|
45
|
+
|
46
|
+
# for a library or gem, you might want to ignore these files since the code is
|
47
|
+
# intended to run in multiple environments; otherwise, check them in:
|
48
|
+
# Gemfile.lock
|
49
|
+
# .ruby-version
|
50
|
+
# .ruby-gemset
|
51
|
+
|
52
|
+
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
53
|
+
.rvmrc
|
data/.gitmodules
ADDED
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--color
|
data/CHANGLOG.md
ADDED
File without changes
|
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
# Qian
|
data/Rakefile
ADDED
data/lib/qian.rb
ADDED
@@ -0,0 +1,123 @@
|
|
1
|
+
require "avro"
|
2
|
+
require "kafka"
|
3
|
+
require "avro_turf/messaging"
|
4
|
+
require "virtus"
|
5
|
+
require "qian/util"
|
6
|
+
require "qian/event"
|
7
|
+
require "qian/logger"
|
8
|
+
require "active_support"
|
9
|
+
require "active_support/core_ext"
|
10
|
+
require "qian/events/engagement"
|
11
|
+
|
12
|
+
module Qian
|
13
|
+
extend self
|
14
|
+
|
15
|
+
attr_accessor :logger_config
|
16
|
+
attr_accessor :kafka_config
|
17
|
+
attr_accessor :schema_registry_url
|
18
|
+
attr_accessor :schema_path
|
19
|
+
attr_accessor :disabled
|
20
|
+
|
21
|
+
#
|
22
|
+
# 当前 Qian lib 的路径
|
23
|
+
#
|
24
|
+
#
|
25
|
+
# @return [Pathname]
|
26
|
+
#
|
27
|
+
def root
|
28
|
+
Pathname.new(File.expand_path(File.dirname(__FILE__)))
|
29
|
+
end
|
30
|
+
|
31
|
+
#
|
32
|
+
# 设置 Qian 配置
|
33
|
+
#
|
34
|
+
#
|
35
|
+
# @return []
|
36
|
+
#
|
37
|
+
def configure(&blk)
|
38
|
+
blk.yield(Qian)
|
39
|
+
|
40
|
+
#
|
41
|
+
# Setting up
|
42
|
+
#
|
43
|
+
@kafka ||= Kafka.new(kafka_config[:connection])
|
44
|
+
@avro ||= AvroTurf::Messaging.new(
|
45
|
+
:registry_url => schema_registry_url, :schemas_path => schema_path)
|
46
|
+
end
|
47
|
+
|
48
|
+
#
|
49
|
+
# 是否禁用
|
50
|
+
#
|
51
|
+
#
|
52
|
+
# @return [Boolean]
|
53
|
+
#
|
54
|
+
def disabled?
|
55
|
+
self.disabled
|
56
|
+
end
|
57
|
+
|
58
|
+
#
|
59
|
+
# 返回一个 Kafka Async Producer 实例
|
60
|
+
#
|
61
|
+
#
|
62
|
+
# @return [Kafka::AsyncProducer]
|
63
|
+
#
|
64
|
+
def kafka_producer
|
65
|
+
return nil if disabled?
|
66
|
+
@producer ||= @kafka.async_producer(kafka_config[:async_config])
|
67
|
+
end
|
68
|
+
|
69
|
+
#
|
70
|
+
# 返回 Qian::Logger
|
71
|
+
#
|
72
|
+
#
|
73
|
+
# @return [Qian::Logger]
|
74
|
+
#
|
75
|
+
def logger
|
76
|
+
@logger || Qian::Logger.new(kafka_producer)
|
77
|
+
end
|
78
|
+
|
79
|
+
#
|
80
|
+
# 返回 Avro Instance
|
81
|
+
#
|
82
|
+
#
|
83
|
+
# @return [Avro]
|
84
|
+
#
|
85
|
+
def avro
|
86
|
+
@avro
|
87
|
+
end
|
88
|
+
|
89
|
+
#
|
90
|
+
# 资源清理
|
91
|
+
#
|
92
|
+
#
|
93
|
+
# @return [void]
|
94
|
+
#
|
95
|
+
def shutdown
|
96
|
+
# 关闭 Kafka Producer
|
97
|
+
@producer.present? && @producer.shutdown
|
98
|
+
end
|
99
|
+
|
100
|
+
#
|
101
|
+
# Default Settings
|
102
|
+
#
|
103
|
+
self.kafka_config = {
|
104
|
+
:connection => {
|
105
|
+
:seed_brokers => ["localhost:9092"],
|
106
|
+
:client_id => "kafka-rb-producer"
|
107
|
+
},
|
108
|
+
:async => {
|
109
|
+
:delivery_threshold => 2_000,
|
110
|
+
:delivery_interval => 30,
|
111
|
+
:max_buffer_size => 2_000,
|
112
|
+
:max_buffer_bytesize => 100_000_000
|
113
|
+
}
|
114
|
+
}
|
115
|
+
|
116
|
+
self.logger_config = {
|
117
|
+
:topic => "admin-app-log"
|
118
|
+
}
|
119
|
+
|
120
|
+
self.schema_registry_url = "http://127.0.0.1:28081/"
|
121
|
+
self.schema_path = Qian.root.join("..", "schema")
|
122
|
+
self.disabled = false
|
123
|
+
end
|
data/lib/qian/event.rb
ADDED
@@ -0,0 +1,71 @@
|
|
1
|
+
module Qian
|
2
|
+
module Event
|
3
|
+
|
4
|
+
def self.included(base)
|
5
|
+
base.class_eval do
|
6
|
+
include Virtus.value_object(:strict => true)
|
7
|
+
|
8
|
+
#
|
9
|
+
# 设置当前 Event 的 Kafka Topic
|
10
|
+
#
|
11
|
+
# @param [<type>] topic_name <description>
|
12
|
+
#
|
13
|
+
# @return [<type>] <description>
|
14
|
+
#
|
15
|
+
def self.kafka_topic(topic_name)
|
16
|
+
@kafka_topic_name = topic_name.to_s
|
17
|
+
end
|
18
|
+
|
19
|
+
#
|
20
|
+
# 返回当前 Event 的 Kafka Topic
|
21
|
+
#
|
22
|
+
# @return [String]
|
23
|
+
#
|
24
|
+
def self.kafka_topic_name
|
25
|
+
@kafka_topic_name
|
26
|
+
end
|
27
|
+
|
28
|
+
#
|
29
|
+
# 当前 Event 类型对应的 Avro Schema 全名
|
30
|
+
#
|
31
|
+
#
|
32
|
+
# @return [String]
|
33
|
+
#
|
34
|
+
def self.avro_schema_name
|
35
|
+
"com.jianshu.event.#{Qian::Util.convert_class_name_to_package_name(self.to_s)}"
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
#
|
41
|
+
# 将自己事件发送出去
|
42
|
+
#
|
43
|
+
#
|
44
|
+
# @return [void]
|
45
|
+
#
|
46
|
+
def emit!
|
47
|
+
Qian.kafka_producer.produce(avro_encoded_data, :topic => self.class.kafka_topic_name)
|
48
|
+
end
|
49
|
+
|
50
|
+
#
|
51
|
+
# 将自己 encode 为 avro binary data
|
52
|
+
#
|
53
|
+
# * Avro::Turf 只接受
|
54
|
+
#
|
55
|
+
# @return [String]
|
56
|
+
#
|
57
|
+
def attrs_with_string_key
|
58
|
+
self.attributes.deep_stringify_keys
|
59
|
+
end
|
60
|
+
|
61
|
+
#
|
62
|
+
# 将自己 encode 为 avro binary data
|
63
|
+
#
|
64
|
+
#
|
65
|
+
# @return [String]
|
66
|
+
#
|
67
|
+
def avro_encoded_data
|
68
|
+
Qian.avro.encode(self.attrs_with_string_key, :schema_name => self.class.avro_schema_name)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Qian
|
2
|
+
module Events
|
3
|
+
class Engagement
|
4
|
+
include Qian::Event
|
5
|
+
|
6
|
+
kafka_topic :qian_engagement_events
|
7
|
+
|
8
|
+
#
|
9
|
+
# Attributes
|
10
|
+
#
|
11
|
+
attribute :user_id, Integer
|
12
|
+
attribute :action, String
|
13
|
+
attribute :server_time, Integer
|
14
|
+
attribute :local_time, Integer
|
15
|
+
attribute :placement, Integer
|
16
|
+
attribute :props, Hash[String => String]
|
17
|
+
attribute :app, String
|
18
|
+
attribute :app_version, String
|
19
|
+
attribute :real_ip, String
|
20
|
+
attribute :batch_id, String
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
data/lib/qian/logger.rb
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
module Qian
|
2
|
+
#
|
3
|
+
# 发送 json 数据到 Kafka 的 Logger
|
4
|
+
#
|
5
|
+
# @author Larry <larry@jianshu.com>
|
6
|
+
#
|
7
|
+
class Logger
|
8
|
+
def initialize(kafka_producer)
|
9
|
+
@kafka_producer = kafka_producer
|
10
|
+
end
|
11
|
+
|
12
|
+
#
|
13
|
+
# Log
|
14
|
+
#
|
15
|
+
# @param [Hash] entry Log 到 Kafka 的数据
|
16
|
+
# @param [Hash] options 额外的参数
|
17
|
+
#
|
18
|
+
# @return [void]
|
19
|
+
#
|
20
|
+
def log(entry = {}, options = {})
|
21
|
+
return if entry.empty? || @kafka_producer.nil?
|
22
|
+
topic = options[:topic] || Qian.logger_config[:topic]
|
23
|
+
@kafka_producer.produce(entry.to_json, :topic => topic)
|
24
|
+
end
|
25
|
+
|
26
|
+
#
|
27
|
+
# 批量 Log
|
28
|
+
#
|
29
|
+
# @param [Array] entries
|
30
|
+
# @param [Hash] options
|
31
|
+
#
|
32
|
+
# @return [<type>] <description>
|
33
|
+
#
|
34
|
+
def log_in_batch(entries = [], options = {})
|
35
|
+
return if entries.empty? || @kafka_producer.nil?
|
36
|
+
entries.each { |entry| log(entry, options) }
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
data/lib/qian/util.rb
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
module Qian
|
2
|
+
module Util
|
3
|
+
extend self
|
4
|
+
|
5
|
+
#
|
6
|
+
# 将 Ruby 的 Class 字符串转为类 Java 的包名
|
7
|
+
#
|
8
|
+
# @param [String] class_name
|
9
|
+
#
|
10
|
+
# @return [String]
|
11
|
+
#
|
12
|
+
def convert_class_name_to_package_name(class_name)
|
13
|
+
segments = class_name.downcase.split("::")
|
14
|
+
segments[segments.length - 1] = segments[segments.length - 1].capitalize
|
15
|
+
segments.join(".")
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
data/lib/qian/version.rb
ADDED
data/qian.gemspec
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
lib = File.expand_path('../lib', __FILE__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
require 'qian/version'
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = "qian"
|
7
|
+
spec.version = Qian::VERSION
|
8
|
+
spec.authors = ["Larry Zhao"]
|
9
|
+
spec.email = ["larry@jianshu.com"]
|
10
|
+
spec.summary = %q{Client to broadcast and log.}
|
11
|
+
spec.description = %q{}
|
12
|
+
spec.homepage = ""
|
13
|
+
spec.license = "MIT"
|
14
|
+
|
15
|
+
spec.files = `git ls-files -z`.split("\x0")
|
16
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
17
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
|
+
spec.require_paths = ["lib"]
|
19
|
+
|
20
|
+
spec.required_ruby_version = ">= 2.3.0"
|
21
|
+
|
22
|
+
spec.add_dependency "ruby-kafka", ">= 0.5.1"
|
23
|
+
spec.add_dependency "avro_turf", ">= 0.8.0"
|
24
|
+
spec.add_dependency "virtus", ">= 1.0.5"
|
25
|
+
spec.add_dependency "activesupport", ">= 5.0.5"
|
26
|
+
|
27
|
+
spec.add_development_dependency "bundler", "~> 1.5"
|
28
|
+
spec.add_development_dependency "rake"
|
29
|
+
spec.add_development_dependency "rspec", "~> 3.5.0"
|
30
|
+
spec.add_development_dependency "pry"
|
31
|
+
spec.add_development_dependency "pry-byebug"
|
32
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
module Test
|
4
|
+
class Engagement
|
5
|
+
include Qian::Event
|
6
|
+
kafka_topic(:test_topic)
|
7
|
+
|
8
|
+
attribute :user_id, Integer
|
9
|
+
attribute :action, String
|
10
|
+
attribute :server_time, Integer
|
11
|
+
attribute :local_time, Integer
|
12
|
+
attribute :placement, Integer
|
13
|
+
attribute :props, Hash[String => String]
|
14
|
+
attribute :app, String
|
15
|
+
attribute :app_version, String
|
16
|
+
attribute :real_ip, String
|
17
|
+
attribute :batch_id, String
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
describe Qian::Event do
|
22
|
+
before(:all) do
|
23
|
+
kafka_config = {
|
24
|
+
:connection => {
|
25
|
+
:seed_brokers => ["10.0.0.1:9092"],
|
26
|
+
:client_id => "kafka-client-id"
|
27
|
+
},
|
28
|
+
:async_config => {
|
29
|
+
:delivery_threshold => 100,
|
30
|
+
:delivery_interval => 200,
|
31
|
+
:max_buffer_size => 300,
|
32
|
+
:max_buffer_bytesize => 400
|
33
|
+
}
|
34
|
+
}
|
35
|
+
|
36
|
+
schema_registry_url = "http://127.0.0.1:28081"
|
37
|
+
|
38
|
+
Qian.configure do |config|
|
39
|
+
config.kafka_config = kafka_config
|
40
|
+
config.schema_registry_url = schema_registry_url
|
41
|
+
config.schema_path = File.join(SPEC_ROOT, "support", "avro_schema")
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
describe ".kakfa_topic" do
|
46
|
+
it "should correctly set topic name" do
|
47
|
+
expect(Test::Engagement.kafka_topic_name).to eq("test_topic")
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
describe ".avro_schema_name" do
|
52
|
+
it "should return correct avro schema name" do
|
53
|
+
expect(Test::Engagement.avro_schema_name).to eq("com.jianshu.event.test.Engagement")
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
describe "#avro_encoded_data" do
|
58
|
+
it "should return encoded data" do
|
59
|
+
engagement_data = {
|
60
|
+
"user_id" => 233,
|
61
|
+
"action" => "IMPRESSION",
|
62
|
+
"server_time" => 10000,
|
63
|
+
"local_time" => 20000,
|
64
|
+
"props" => {
|
65
|
+
"item_type" => "Note",
|
66
|
+
"item_id" => "3344"
|
67
|
+
},
|
68
|
+
"app" => "M7E",
|
69
|
+
"app_version" => "4.3.0",
|
70
|
+
"placement" => 5,
|
71
|
+
"real_ip" => "291.168.1.1",
|
72
|
+
"batch_id" => SecureRandom.uuid
|
73
|
+
}
|
74
|
+
|
75
|
+
engagement = Test::Engagement.new(engagement_data)
|
76
|
+
|
77
|
+
data = engagement.avro_encoded_data
|
78
|
+
|
79
|
+
expect(data).not_to be_nil
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
describe "#emit!" do
|
84
|
+
it "should emit event to kafka" do
|
85
|
+
engagement_data = {
|
86
|
+
"user_id" => 233,
|
87
|
+
"action" => "CLICK",
|
88
|
+
"server_time" => 10000,
|
89
|
+
"local_time" => 20000,
|
90
|
+
"props" => {
|
91
|
+
"item_type" => "Note",
|
92
|
+
"item_id" => "3344"
|
93
|
+
},
|
94
|
+
"app"=> "HUGO",
|
95
|
+
"placement"=> 5,
|
96
|
+
"app_version"=> "4.3.0",
|
97
|
+
"real_ip"=> "291.168.1.1",
|
98
|
+
"batch_id" => SecureRandom.uuid
|
99
|
+
}
|
100
|
+
|
101
|
+
engagement = Test::Engagement.new(engagement_data)
|
102
|
+
|
103
|
+
avro_data = "dummy_avro_data"
|
104
|
+
expect(engagement).to receive(:avro_encoded_data).and_return(avro_data)
|
105
|
+
|
106
|
+
producer = double
|
107
|
+
expect(Qian).to receive(:kafka_producer).and_return(producer)
|
108
|
+
expect(producer).to receive(:produce).with(avro_data, :topic => "test_topic")
|
109
|
+
|
110
|
+
engagement.emit!
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
describe Qian::Logger do
|
4
|
+
before(:all) do
|
5
|
+
kafka_config = {
|
6
|
+
:connection => {
|
7
|
+
:seed_brokers => ["10.0.0.1:9092"],
|
8
|
+
:client_id => "kafka-client-id"
|
9
|
+
},
|
10
|
+
:async_config => {
|
11
|
+
:delivery_threshold => 100,
|
12
|
+
:delivery_interval => 200,
|
13
|
+
:max_buffer_size => 300,
|
14
|
+
:max_buffer_bytesize => 400
|
15
|
+
}
|
16
|
+
}
|
17
|
+
|
18
|
+
schema_registry_url = "http://127.0.0.1:28081"
|
19
|
+
|
20
|
+
Qian.configure do |config|
|
21
|
+
config.kafka_config = kafka_config
|
22
|
+
config.schema_registry_url = schema_registry_url
|
23
|
+
config.schema_path = File.join(SPEC_ROOT, "support", "avro_schema")
|
24
|
+
config.logger_config = { :topic => "dummy_topic" }
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
describe "#log" do
|
29
|
+
it "should send log data as json to topic configured in logger_conifg" do
|
30
|
+
log_data = { :foo => :bar, :hello => :world }
|
31
|
+
|
32
|
+
logger = Qian.logger
|
33
|
+
producer = logger.instance_variable_get("@kafka_producer")
|
34
|
+
|
35
|
+
expect(producer).to receive(:produce).with(log_data.to_json, :topic => "dummy_topic" )
|
36
|
+
Qian.logger.log(log_data)
|
37
|
+
end
|
38
|
+
|
39
|
+
it "should send log data as json to topic pass in arguments" do
|
40
|
+
log_data = { :foo => :bar, :hello => :world }
|
41
|
+
|
42
|
+
logger = Qian.logger
|
43
|
+
producer = logger.instance_variable_get("@kafka_producer")
|
44
|
+
|
45
|
+
expect(producer).to receive(:produce).with(log_data.to_json, :topic => "some_topic" )
|
46
|
+
Qian.logger.log(log_data, :topic => "some_topic")
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
data/spec/qian_spec.rb
ADDED
@@ -0,0 +1,62 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
describe Qian do
|
4
|
+
let(:kafka_config) do
|
5
|
+
{
|
6
|
+
:connection => {
|
7
|
+
:seed_brokers => ["10.0.0.1:9092"],
|
8
|
+
:client_id => "kafka-client-id"
|
9
|
+
},
|
10
|
+
:async_config => {
|
11
|
+
:delivery_threshold => 100,
|
12
|
+
:delivery_interval => 200,
|
13
|
+
:max_buffer_size => 300,
|
14
|
+
:max_buffer_bytesize => 400
|
15
|
+
}
|
16
|
+
}
|
17
|
+
end
|
18
|
+
|
19
|
+
let(:schema_registry_url) { "http://1.2.3.4:5000" }
|
20
|
+
|
21
|
+
describe "#shutdown" do
|
22
|
+
it "should call shutdown on kafka producer if kafka producer present" do
|
23
|
+
producer = double
|
24
|
+
Qian.instance_variable_set(:@producer, producer)
|
25
|
+
|
26
|
+
expect(producer).to receive(:shutdown)
|
27
|
+
Qian.shutdown
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
describe "#configure" do
|
32
|
+
it "should correctly set config data" do
|
33
|
+
Qian.configure do |config|
|
34
|
+
config.kafka_config = kafka_config
|
35
|
+
config.schema_registry_url = schema_registry_url
|
36
|
+
end
|
37
|
+
|
38
|
+
expect(Qian.kafka_config).to eq(kafka_config)
|
39
|
+
expect(Qian.schema_registry_url).to eq("http://1.2.3.4:5000")
|
40
|
+
expect(Qian.kafka_producer).not_to be_nil
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
describe "#root" do
|
45
|
+
it "should return the path of Qian" do
|
46
|
+
root_path = Qian.root
|
47
|
+
expect(root_path).to be_a(Pathname)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
describe "#logger" do
|
52
|
+
it "should return a Qian::Logger" do
|
53
|
+
logger = Qian.logger
|
54
|
+
|
55
|
+
producer = double
|
56
|
+
|
57
|
+
|
58
|
+
expect(logger).to be_a(Qian::Logger)
|
59
|
+
expect(logger.instance_variable_get("@kafka_producer")).to eq(Qian.instance_variable_get("@producer"))
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
require "active_support"
|
2
|
+
require "qian"
|
3
|
+
require "rspec"
|
4
|
+
|
5
|
+
RSpec.configure do |config|
|
6
|
+
config.expect_with :rspec do |expectations|
|
7
|
+
expectations.syntax = :expect
|
8
|
+
end
|
9
|
+
|
10
|
+
config.mock_with :rspec do |mocks|
|
11
|
+
mocks.syntax = :expect
|
12
|
+
mocks.verify_partial_doubles = true
|
13
|
+
end
|
14
|
+
|
15
|
+
config.order = :random
|
16
|
+
end
|
17
|
+
|
18
|
+
SPEC_ROOT = File.expand_path(File.dirname(__FILE__))
|
@@ -0,0 +1,58 @@
|
|
1
|
+
{
|
2
|
+
"namespace": "com.jianshu.event.test",
|
3
|
+
"type": "record",
|
4
|
+
"name": "Engagement",
|
5
|
+
"fields": [
|
6
|
+
{
|
7
|
+
"name": "user_id",
|
8
|
+
"type": ["int", "null"]
|
9
|
+
},
|
10
|
+
{
|
11
|
+
"name": "action",
|
12
|
+
"type": {
|
13
|
+
"type": "enum",
|
14
|
+
"name": "EngagementAction",
|
15
|
+
"symbols": ["IMPRESSION", "CLICK", "NOT_INTERESTED"]
|
16
|
+
}
|
17
|
+
},
|
18
|
+
{
|
19
|
+
"name": "server_time",
|
20
|
+
"type": "int"
|
21
|
+
},
|
22
|
+
{
|
23
|
+
"name": "local_time",
|
24
|
+
"type": "int"
|
25
|
+
},
|
26
|
+
{
|
27
|
+
"name": "props",
|
28
|
+
"type": {
|
29
|
+
"type": "map",
|
30
|
+
"values": "string"
|
31
|
+
}
|
32
|
+
},
|
33
|
+
{
|
34
|
+
"name": "placement",
|
35
|
+
"type": "int"
|
36
|
+
},
|
37
|
+
{
|
38
|
+
"name": "app",
|
39
|
+
"type": {
|
40
|
+
"type": "enum",
|
41
|
+
"name": "App",
|
42
|
+
"symbols": ["HUGO", "HARUKI", "VICTOR", "M7E", "M7E_H5"]
|
43
|
+
}
|
44
|
+
},
|
45
|
+
{
|
46
|
+
"name": "app_version",
|
47
|
+
"type": "string"
|
48
|
+
},
|
49
|
+
{
|
50
|
+
"name": "real_ip",
|
51
|
+
"type": "string"
|
52
|
+
},
|
53
|
+
{
|
54
|
+
"name": "batch_id",
|
55
|
+
"type": "string"
|
56
|
+
}
|
57
|
+
]
|
58
|
+
}
|
metadata
ADDED
@@ -0,0 +1,196 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: qian
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.11
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Larry Zhao
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-12-27 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: ruby-kafka
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.5.1
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.5.1
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: avro_turf
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.8.0
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 0.8.0
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: virtus
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 1.0.5
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 1.0.5
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: activesupport
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 5.0.5
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 5.0.5
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: bundler
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '1.5'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '1.5'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rake
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rspec
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 3.5.0
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: 3.5.0
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: pry
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: pry-byebug
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
139
|
+
description: ''
|
140
|
+
email:
|
141
|
+
- larry@jianshu.com
|
142
|
+
executables: []
|
143
|
+
extensions: []
|
144
|
+
extra_rdoc_files: []
|
145
|
+
files:
|
146
|
+
- ".gitignore"
|
147
|
+
- ".gitmodules"
|
148
|
+
- ".rspec"
|
149
|
+
- CHANGLOG.md
|
150
|
+
- Gemfile
|
151
|
+
- README.md
|
152
|
+
- Rakefile
|
153
|
+
- lib/qian.rb
|
154
|
+
- lib/qian/event.rb
|
155
|
+
- lib/qian/events/engagement.rb
|
156
|
+
- lib/qian/logger.rb
|
157
|
+
- lib/qian/util.rb
|
158
|
+
- lib/qian/version.rb
|
159
|
+
- qian.gemspec
|
160
|
+
- spec/qian/event_spec.rb
|
161
|
+
- spec/qian/logger_spec.rb
|
162
|
+
- spec/qian/util_spec.rb
|
163
|
+
- spec/qian_spec.rb
|
164
|
+
- spec/spec_helper.rb
|
165
|
+
- spec/support/avro_schema/com/jianshu/event/test/Engagement.avsc
|
166
|
+
homepage: ''
|
167
|
+
licenses:
|
168
|
+
- MIT
|
169
|
+
metadata: {}
|
170
|
+
post_install_message:
|
171
|
+
rdoc_options: []
|
172
|
+
require_paths:
|
173
|
+
- lib
|
174
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
175
|
+
requirements:
|
176
|
+
- - ">="
|
177
|
+
- !ruby/object:Gem::Version
|
178
|
+
version: 2.3.0
|
179
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
180
|
+
requirements:
|
181
|
+
- - ">="
|
182
|
+
- !ruby/object:Gem::Version
|
183
|
+
version: '0'
|
184
|
+
requirements: []
|
185
|
+
rubyforge_project:
|
186
|
+
rubygems_version: 2.6.14
|
187
|
+
signing_key:
|
188
|
+
specification_version: 4
|
189
|
+
summary: Client to broadcast and log.
|
190
|
+
test_files:
|
191
|
+
- spec/qian/event_spec.rb
|
192
|
+
- spec/qian/logger_spec.rb
|
193
|
+
- spec/qian/util_spec.rb
|
194
|
+
- spec/qian_spec.rb
|
195
|
+
- spec/spec_helper.rb
|
196
|
+
- spec/support/avro_schema/com/jianshu/event/test/Engagement.avsc
|