franz 2.1.5 → 2.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Readme.md +11 -2
- data/VERSION +1 -1
- data/bin/franz +13 -3
- data/lib/franz/output.rb +2 -1
- data/lib/franz/output/{stdout.rb → device.rb} +6 -3
- data/lib/franz/output/http.rb +128 -0
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 192d0de743aeb1560eb55145d11b06c2f1f5ba61
|
4
|
+
data.tar.gz: 145887b6c3313ad29f4eb88f9d17df62899a27c2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b2a4cde46ea536888c90c4f2fb2611a7f4526840c2a422d85f99b646385b580e48f1eee92fc91c0422c1287d075d8e92530c80f060e0bdeb65cbace48276517f
|
7
|
+
data.tar.gz: fdbb262b5620837f14d92986e12d946decd4b335665953ff7784151dcf6fafee8ba0843cbe14179d9a9e141cdbb2fdabe8ce187043ebc02db3d0ef0ac21787b0
|
data/Readme.md
CHANGED
@@ -188,8 +188,15 @@ It's kinda like a JSON version of the Logstash config language:
|
|
188
188
|
"socket_timeout_ms": 10000
|
189
189
|
},
|
190
190
|
|
191
|
-
//
|
192
|
-
"
|
191
|
+
// HTTP (experimental)
|
192
|
+
"http": {
|
193
|
+
"server": "http://localhost:3000",
|
194
|
+
"flush_interval": 5,
|
195
|
+
"flush_size": 500
|
196
|
+
},
|
197
|
+
|
198
|
+
// Device (STDOUT implied if neither Kafka nor RabbitMQ configured)
|
199
|
+
"device": "/dev/stdout",
|
193
200
|
|
194
201
|
// Advanced configuration (optional)
|
195
202
|
"stats_interval": 60, // Emit statistics periodically
|
@@ -223,8 +230,10 @@ for Chef.
|
|
223
230
|
- Self-contained packages for OS X and Linux thanks to [Traveling Ruby](http://phusion.github.io/traveling-ruby) and [FPM](https://github.com/jordansissel/fpm)
|
224
231
|
- Dockerization using self-contained Linux package [available on the Docker Hub](https://registry.hub.docker.com/u/sczizzo/franz)
|
225
232
|
- Running Franz without arguments now prints help text (i.e. `--config` is now required)
|
233
|
+
- Rename `StdOut` to `Device`, no longer experimental
|
226
234
|
- Allow single state file if no glob is used
|
227
235
|
- Handle `EACCESS` on `IO.read` gracefully
|
236
|
+
- New `HTTP` output (experimental)
|
228
237
|
- Remove default line limit
|
229
238
|
|
230
239
|
#### v2.0
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.1.
|
1
|
+
2.1.6
|
data/bin/franz
CHANGED
@@ -41,10 +41,11 @@ config = Franz::Config.new opts[:config]
|
|
41
41
|
level = :info
|
42
42
|
level = :debug if opts[:debug]
|
43
43
|
level = :trace if opts[:trace]
|
44
|
+
device = opts[:log] || $stdout
|
44
45
|
colorize, prettify = false, false
|
45
|
-
colorize, prettify = true, true if
|
46
|
+
colorize, prettify = true, true if device.tty? rescue false
|
46
47
|
logger = Slog.new (config[:slog] || {}).merge \
|
47
|
-
out:
|
48
|
+
out: device,
|
48
49
|
colorize: colorize,
|
49
50
|
prettify: prettify,
|
50
51
|
level: level
|
@@ -81,10 +82,19 @@ elsif config[:output][:kafka]
|
|
81
82
|
tags: config[:output][:tags],
|
82
83
|
statz: statz
|
83
84
|
|
85
|
+
elsif config[:output][:http]
|
86
|
+
Franz::Output::HTTP.new \
|
87
|
+
input: io,
|
88
|
+
output: config[:output][:http],
|
89
|
+
logger: logger,
|
90
|
+
tags: config[:output][:tags],
|
91
|
+
statz: statz
|
92
|
+
|
84
93
|
else
|
85
|
-
Franz::Output::
|
94
|
+
Franz::Output::Device.new \
|
86
95
|
input: io,
|
87
96
|
logger: logger,
|
97
|
+
output: config[:output][:device],
|
88
98
|
tags: config[:output][:tags],
|
89
99
|
statz: statz
|
90
100
|
end
|
data/lib/franz/output.rb
CHANGED
@@ -7,7 +7,7 @@ module Franz
|
|
7
7
|
module Output
|
8
8
|
|
9
9
|
# STDOUT output for Franz.
|
10
|
-
class
|
10
|
+
class Device
|
11
11
|
|
12
12
|
# Start a new output in the background. We'll consume from the input queue
|
13
13
|
# and ship events to STDOUT.
|
@@ -18,12 +18,14 @@ module Franz
|
|
18
18
|
opts = {
|
19
19
|
logger: Logger.new(STDOUT),
|
20
20
|
tags: [],
|
21
|
-
input: []
|
21
|
+
input: [],
|
22
|
+
output: '/dev/stdout'
|
22
23
|
}.deep_merge!(opts)
|
23
24
|
|
24
25
|
@statz = opts[:statz] || Franz::Stats.new
|
25
26
|
@statz.create :num_output, 0
|
26
27
|
|
28
|
+
@device = File.open(opts[:output], 'w')
|
27
29
|
@logger = opts[:logger]
|
28
30
|
|
29
31
|
@stop = false
|
@@ -37,7 +39,7 @@ module Franz
|
|
37
39
|
event: 'publish',
|
38
40
|
raw: event
|
39
41
|
|
40
|
-
puts JSON::generate(event)
|
42
|
+
@device.puts JSON::generate(event)
|
41
43
|
|
42
44
|
@statz.inc :num_output
|
43
45
|
end
|
@@ -60,6 +62,7 @@ module Franz
|
|
60
62
|
return if @foreground
|
61
63
|
@foreground = true
|
62
64
|
@thread.kill
|
65
|
+
@device.close
|
63
66
|
log.info event: 'output stopped'
|
64
67
|
end
|
65
68
|
|
@@ -0,0 +1,128 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'thread'
|
3
|
+
require 'fiber'
|
4
|
+
require 'json'
|
5
|
+
|
6
|
+
require 'deep_merge'
|
7
|
+
|
8
|
+
|
9
|
+
module Franz
|
10
|
+
module Output
|
11
|
+
|
12
|
+
# HTTP output for Franz.
|
13
|
+
class HTTP
|
14
|
+
# Start a new output in the background. We'll consume from the input queue
|
15
|
+
# and ship events via HTTP.
|
16
|
+
#
|
17
|
+
# @param [Hash] opts options for the output
|
18
|
+
# @option opts [Queue] :input ([]) "input" queue
|
19
|
+
# @option opts [Queue] :output ([]) "output" configuration
|
20
|
+
def initialize opts={}
|
21
|
+
opts = {
|
22
|
+
logger: Logger.new(STDOUT),
|
23
|
+
tags: [],
|
24
|
+
input: [],
|
25
|
+
output: {
|
26
|
+
server: 'http://localhost:3000',
|
27
|
+
flush_size: 500,
|
28
|
+
flush_interval: 10
|
29
|
+
}
|
30
|
+
}.deep_merge!(opts)
|
31
|
+
|
32
|
+
@statz = opts[:statz] || Franz::Stats.new
|
33
|
+
@statz.create :num_output, 0
|
34
|
+
|
35
|
+
@logger = opts[:logger]
|
36
|
+
|
37
|
+
@stop = false
|
38
|
+
@foreground = opts[:foreground]
|
39
|
+
|
40
|
+
server = opts[:output].delete :server
|
41
|
+
@uri = URI(server)
|
42
|
+
open_uri
|
43
|
+
|
44
|
+
@flush_size = opts[:output][:flush_size]
|
45
|
+
@flush_interval = opts[:output][:flush_interval]
|
46
|
+
@lock = Mutex.new
|
47
|
+
@messages = []
|
48
|
+
|
49
|
+
Thread.new do
|
50
|
+
until @stop
|
51
|
+
@lock.synchronize do
|
52
|
+
flush_messages true
|
53
|
+
end
|
54
|
+
sleep @flush_interval
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
@thread = Thread.new do
|
59
|
+
until @stop
|
60
|
+
event = JSON::generate(opts[:input].shift)
|
61
|
+
@lock.synchronize do
|
62
|
+
enqueue event
|
63
|
+
end
|
64
|
+
|
65
|
+
log.trace \
|
66
|
+
event: 'publish',
|
67
|
+
raw: event
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
log.info event: 'output started'
|
72
|
+
|
73
|
+
@thread.join if @foreground
|
74
|
+
end
|
75
|
+
|
76
|
+
|
77
|
+
# Join the Output thread. Effectively only once.
|
78
|
+
def join
|
79
|
+
return if @foreground
|
80
|
+
@foreground = true
|
81
|
+
@thread.join
|
82
|
+
end
|
83
|
+
|
84
|
+
|
85
|
+
# Stop the Output thread. Effectively only once.
|
86
|
+
def stop
|
87
|
+
return if @foreground
|
88
|
+
@foreground = true
|
89
|
+
@thread.kill
|
90
|
+
log.info event: 'output stopped'
|
91
|
+
end
|
92
|
+
|
93
|
+
|
94
|
+
private
|
95
|
+
def log ; @logger end
|
96
|
+
|
97
|
+
def open_uri
|
98
|
+
@http = Net::HTTP.new(@uri.host, @uri.port)
|
99
|
+
end
|
100
|
+
|
101
|
+
def enqueue event
|
102
|
+
@messages << event
|
103
|
+
flush_messages
|
104
|
+
end
|
105
|
+
|
106
|
+
def flush_messages force=false
|
107
|
+
size = @messages.length
|
108
|
+
return if size.zero?
|
109
|
+
if force || size >= @flush_size
|
110
|
+
emit @messages.join("\n")
|
111
|
+
@statz.inc :num_output, size
|
112
|
+
@messages.clear
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def emit body
|
117
|
+
request = Net::HTTP::Post.new(@uri)
|
118
|
+
request.body = body
|
119
|
+
@http.request(request)
|
120
|
+
rescue EOFError, Errno::ECONNREFUSED, Errno::EPIPE
|
121
|
+
log.warn event: 'output reconnect'
|
122
|
+
open_uri
|
123
|
+
sleep 1
|
124
|
+
retry
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: franz
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.1.
|
4
|
+
version: 2.1.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sean Clemmer
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-05-
|
11
|
+
date: 2015-05-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: slog
|
@@ -141,9 +141,10 @@ files:
|
|
141
141
|
- lib/franz/input_config.rb
|
142
142
|
- lib/franz/metadata.rb
|
143
143
|
- lib/franz/output.rb
|
144
|
+
- lib/franz/output/device.rb
|
145
|
+
- lib/franz/output/http.rb
|
144
146
|
- lib/franz/output/kafka.rb
|
145
147
|
- lib/franz/output/rabbitmq.rb
|
146
|
-
- lib/franz/output/stdout.rb
|
147
148
|
- lib/franz/sash.rb
|
148
149
|
- lib/franz/stats.rb
|
149
150
|
- lib/franz/tail.rb
|