franz 2.1.5 → 2.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 1a45e8b1c89419f9b75b0b04fde00fddedb4ac0a
4
- data.tar.gz: 011ccd003599089f514a93d0124224c22911d38d
3
+ metadata.gz: 192d0de743aeb1560eb55145d11b06c2f1f5ba61
4
+ data.tar.gz: 145887b6c3313ad29f4eb88f9d17df62899a27c2
5
5
  SHA512:
6
- metadata.gz: c145e7fd8ea9acd1b623bf6ce742bc72ad6adc992c226275c11881f6731b3e2b01bc64fa1e991dfe8e6ff652a074ca261c3900c3a94f2b0301cc7c87ea892bdd
7
- data.tar.gz: a8f94d4ac45dbdffc8e920dc899fa23f8cd9e5941a37c7729caaf53dcf78e7f29034f30ba8b4199a05a506315fa41873f4d28eee6cf624d3f818a6533ab7db75
6
+ metadata.gz: b2a4cde46ea536888c90c4f2fb2611a7f4526840c2a422d85f99b646385b580e48f1eee92fc91c0422c1287d075d8e92530c80f060e0bdeb65cbace48276517f
7
+ data.tar.gz: fdbb262b5620837f14d92986e12d946decd4b335665953ff7784151dcf6fafee8ba0843cbe14179d9a9e141cdbb2fdabe8ce187043ebc02db3d0ef0ac21787b0
data/Readme.md CHANGED
@@ -188,8 +188,15 @@ It's kinda like a JSON version of the Logstash config language:
188
188
  "socket_timeout_ms": 10000
189
189
  },
190
190
 
191
- // StdOut (implied if neither Kafka nor RabbitMQ configured)
192
- "stdout": {},
191
+ // HTTP (experimental)
192
+ "http": {
193
+ "server": "http://localhost:3000",
194
+ "flush_interval": 5,
195
+ "flush_size": 500
196
+ },
197
+
198
+ // Device (STDOUT implied if neither Kafka nor RabbitMQ configured)
199
+ "device": "/dev/stdout",
193
200
 
194
201
  // Advanced configuration (optional)
195
202
  "stats_interval": 60, // Emit statistics periodically
@@ -223,8 +230,10 @@ for Chef.
223
230
  - Self-contained packages for OS X and Linux thanks to [Traveling Ruby](http://phusion.github.io/traveling-ruby) and [FPM](https://github.com/jordansissel/fpm)
224
231
  - Dockerization using self-contained Linux package [available on the Docker Hub](https://registry.hub.docker.com/u/sczizzo/franz)
225
232
  - Running Franz without arguments now prints help text (i.e. `--config` is now required)
233
+ - Rename `StdOut` to `Device`, no longer experimental
226
234
  - Allow single state file if no glob is used
227
235
  - Handle `EACCESS` on `IO.read` gracefully
236
+ - New `HTTP` output (experimental)
228
237
  - Remove default line limit
229
238
 
230
239
  #### v2.0
data/VERSION CHANGED
@@ -1 +1 @@
1
- 2.1.5
1
+ 2.1.6
data/bin/franz CHANGED
@@ -41,10 +41,11 @@ config = Franz::Config.new opts[:config]
41
41
  level = :info
42
42
  level = :debug if opts[:debug]
43
43
  level = :trace if opts[:trace]
44
+ device = opts[:log] || $stdout
44
45
  colorize, prettify = false, false
45
- colorize, prettify = true, true if opts[:log].nil?
46
+ colorize, prettify = true, true if device.tty? rescue false
46
47
  logger = Slog.new (config[:slog] || {}).merge \
47
- out: opts[:log] || $stdout,
48
+ out: device,
48
49
  colorize: colorize,
49
50
  prettify: prettify,
50
51
  level: level
@@ -81,10 +82,19 @@ elsif config[:output][:kafka]
81
82
  tags: config[:output][:tags],
82
83
  statz: statz
83
84
 
85
+ elsif config[:output][:http]
86
+ Franz::Output::HTTP.new \
87
+ input: io,
88
+ output: config[:output][:http],
89
+ logger: logger,
90
+ tags: config[:output][:tags],
91
+ statz: statz
92
+
84
93
  else
85
- Franz::Output::StdOut.new \
94
+ Franz::Output::Device.new \
86
95
  input: io,
87
96
  logger: logger,
97
+ output: config[:output][:device],
88
98
  tags: config[:output][:tags],
89
99
  statz: statz
90
100
  end
data/lib/franz/output.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  require_relative 'stats'
2
+ require_relative 'output/http'
2
3
  require_relative 'output/kafka'
3
- require_relative 'output/stdout'
4
+ require_relative 'output/device'
4
5
  require_relative 'output/rabbitmq'
@@ -7,7 +7,7 @@ module Franz
7
7
  module Output
8
8
 
9
9
  # STDOUT output for Franz.
10
- class StdOut
10
+ class Device
11
11
 
12
12
  # Start a new output in the background. We'll consume from the input queue
13
13
  # and ship events to STDOUT.
@@ -18,12 +18,14 @@ module Franz
18
18
  opts = {
19
19
  logger: Logger.new(STDOUT),
20
20
  tags: [],
21
- input: []
21
+ input: [],
22
+ output: '/dev/stdout'
22
23
  }.deep_merge!(opts)
23
24
 
24
25
  @statz = opts[:statz] || Franz::Stats.new
25
26
  @statz.create :num_output, 0
26
27
 
28
+ @device = File.open(opts[:output], 'w')
27
29
  @logger = opts[:logger]
28
30
 
29
31
  @stop = false
@@ -37,7 +39,7 @@ module Franz
37
39
  event: 'publish',
38
40
  raw: event
39
41
 
40
- puts JSON::generate(event)
42
+ @device.puts JSON::generate(event)
41
43
 
42
44
  @statz.inc :num_output
43
45
  end
@@ -60,6 +62,7 @@ module Franz
60
62
  return if @foreground
61
63
  @foreground = true
62
64
  @thread.kill
65
+ @device.close
63
66
  log.info event: 'output stopped'
64
67
  end
65
68
 
@@ -0,0 +1,128 @@
1
+ require 'net/http'
2
+ require 'thread'
3
+ require 'fiber'
4
+ require 'json'
5
+
6
+ require 'deep_merge'
7
+
8
+
9
+ module Franz
10
+ module Output
11
+
12
+ # HTTP output for Franz.
13
+ class HTTP
14
+ # Start a new output in the background. We'll consume from the input queue
15
+ # and ship events via HTTP.
16
+ #
17
+ # @param [Hash] opts options for the output
18
+ # @option opts [Queue] :input ([]) "input" queue
19
+ # @option opts [Queue] :output ([]) "output" configuration
20
+ def initialize opts={}
21
+ opts = {
22
+ logger: Logger.new(STDOUT),
23
+ tags: [],
24
+ input: [],
25
+ output: {
26
+ server: 'http://localhost:3000',
27
+ flush_size: 500,
28
+ flush_interval: 10
29
+ }
30
+ }.deep_merge!(opts)
31
+
32
+ @statz = opts[:statz] || Franz::Stats.new
33
+ @statz.create :num_output, 0
34
+
35
+ @logger = opts[:logger]
36
+
37
+ @stop = false
38
+ @foreground = opts[:foreground]
39
+
40
+ server = opts[:output].delete :server
41
+ @uri = URI(server)
42
+ open_uri
43
+
44
+ @flush_size = opts[:output][:flush_size]
45
+ @flush_interval = opts[:output][:flush_interval]
46
+ @lock = Mutex.new
47
+ @messages = []
48
+
49
+ Thread.new do
50
+ until @stop
51
+ @lock.synchronize do
52
+ flush_messages true
53
+ end
54
+ sleep @flush_interval
55
+ end
56
+ end
57
+
58
+ @thread = Thread.new do
59
+ until @stop
60
+ event = JSON::generate(opts[:input].shift)
61
+ @lock.synchronize do
62
+ enqueue event
63
+ end
64
+
65
+ log.trace \
66
+ event: 'publish',
67
+ raw: event
68
+ end
69
+ end
70
+
71
+ log.info event: 'output started'
72
+
73
+ @thread.join if @foreground
74
+ end
75
+
76
+
77
+ # Join the Output thread. Effectively only once.
78
+ def join
79
+ return if @foreground
80
+ @foreground = true
81
+ @thread.join
82
+ end
83
+
84
+
85
+ # Stop the Output thread. Effectively only once.
86
+ def stop
87
+ return if @foreground
88
+ @foreground = true
89
+ @thread.kill
90
+ log.info event: 'output stopped'
91
+ end
92
+
93
+
94
+ private
95
+ def log ; @logger end
96
+
97
+ def open_uri
98
+ @http = Net::HTTP.new(@uri.host, @uri.port)
99
+ end
100
+
101
+ def enqueue event
102
+ @messages << event
103
+ flush_messages
104
+ end
105
+
106
+ def flush_messages force=false
107
+ size = @messages.length
108
+ return if size.zero?
109
+ if force || size >= @flush_size
110
+ emit @messages.join("\n")
111
+ @statz.inc :num_output, size
112
+ @messages.clear
113
+ end
114
+ end
115
+
116
+ def emit body
117
+ request = Net::HTTP::Post.new(@uri)
118
+ request.body = body
119
+ @http.request(request)
120
+ rescue EOFError, Errno::ECONNREFUSED, Errno::EPIPE
121
+ log.warn event: 'output reconnect'
122
+ open_uri
123
+ sleep 1
124
+ retry
125
+ end
126
+ end
127
+ end
128
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: franz
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.1.5
4
+ version: 2.1.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sean Clemmer
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-05-20 00:00:00.000000000 Z
11
+ date: 2015-05-26 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: slog
@@ -141,9 +141,10 @@ files:
141
141
  - lib/franz/input_config.rb
142
142
  - lib/franz/metadata.rb
143
143
  - lib/franz/output.rb
144
+ - lib/franz/output/device.rb
145
+ - lib/franz/output/http.rb
144
146
  - lib/franz/output/kafka.rb
145
147
  - lib/franz/output/rabbitmq.rb
146
- - lib/franz/output/stdout.rb
147
148
  - lib/franz/sash.rb
148
149
  - lib/franz/stats.rb
149
150
  - lib/franz/tail.rb