fluq 0.7.5 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.travis.yml +3 -0
- data/Gemfile +12 -1
- data/Gemfile.lock +44 -8
- data/README.md +24 -6
- data/Rakefile +8 -1
- data/benchmark/socket.rb +13 -25
- data/examples/config/multi.rb +52 -0
- data/examples/config/simple.rb +15 -0
- data/fluq.gemspec +3 -3
- data/lib/fluq.rb +22 -16
- data/lib/fluq/cli.rb +3 -12
- data/lib/fluq/dsl.rb +2 -45
- data/lib/fluq/dsl/base.rb +11 -0
- data/lib/fluq/dsl/feed.rb +24 -0
- data/lib/fluq/dsl/root.rb +35 -0
- data/lib/fluq/event.rb +9 -28
- data/lib/fluq/feed.rb +40 -5
- data/lib/fluq/format.rb +6 -0
- data/lib/fluq/format/base.rb +42 -0
- data/lib/fluq/format/json.rb +17 -0
- data/lib/fluq/format/lines.rb +27 -0
- data/lib/fluq/format/msgpack.rb +28 -0
- data/lib/fluq/format/tsv.rb +19 -0
- data/lib/fluq/handler.rb +1 -1
- data/lib/fluq/handler/base.rb +11 -38
- data/lib/fluq/handler/log.rb +12 -14
- data/lib/fluq/handler/noop.rb +2 -0
- data/lib/fluq/input/base.rb +33 -29
- data/lib/fluq/input/socket.rb +46 -16
- data/lib/fluq/mixins.rb +2 -2
- data/lib/fluq/runner.rb +41 -0
- data/lib/fluq/testing.rb +5 -11
- data/lib/fluq/version.rb +1 -1
- data/lib/fluq/worker.rb +73 -0
- data/spec/fluq/dsl/feed_spec.rb +33 -0
- data/spec/fluq/dsl/root_spec.rb +20 -0
- data/spec/fluq/event_spec.rb +17 -12
- data/spec/fluq/feed_spec.rb +24 -0
- data/spec/fluq/format/base_spec.rb +9 -0
- data/spec/fluq/format/json_spec.rb +22 -0
- data/spec/fluq/format/lines_spec.rb +20 -0
- data/spec/fluq/format/msgpack_spec.rb +22 -0
- data/spec/fluq/format/tsv_spec.rb +21 -0
- data/spec/fluq/handler/base_spec.rb +7 -52
- data/spec/fluq/handler/log_spec.rb +11 -14
- data/spec/fluq/handler/{null_spec.rb → noop_spec.rb} +1 -3
- data/spec/fluq/input/base_spec.rb +48 -15
- data/spec/fluq/input/socket_spec.rb +34 -26
- data/spec/fluq/mixins/loggable_spec.rb +2 -2
- data/spec/fluq/runner_spec.rb +18 -0
- data/spec/fluq/worker_spec.rb +87 -0
- data/spec/fluq_spec.rb +1 -2
- data/spec/scenario/config/nested/feed1.rb +6 -0
- data/spec/scenario/config/test.rb +8 -2
- data/spec/spec_helper.rb +7 -26
- metadata +62 -62
- data/benchmark/logging.rb +0 -37
- data/examples/common.rb +0 -3
- data/examples/simple.rb +0 -5
- data/lib/fluq/buffer.rb +0 -6
- data/lib/fluq/buffer/base.rb +0 -51
- data/lib/fluq/buffer/file.rb +0 -68
- data/lib/fluq/feed/base.rb +0 -37
- data/lib/fluq/feed/json.rb +0 -28
- data/lib/fluq/feed/msgpack.rb +0 -27
- data/lib/fluq/feed/tsv.rb +0 -30
- data/lib/fluq/handler/null.rb +0 -4
- data/lib/fluq/input/socket/connection.rb +0 -41
- data/lib/fluq/mixins/logger.rb +0 -26
- data/lib/fluq/reactor.rb +0 -79
- data/spec/fluq/buffer/base_spec.rb +0 -21
- data/spec/fluq/buffer/file_spec.rb +0 -47
- data/spec/fluq/dsl_spec.rb +0 -43
- data/spec/fluq/feed/base_spec.rb +0 -15
- data/spec/fluq/feed/json_spec.rb +0 -27
- data/spec/fluq/feed/msgpack_spec.rb +0 -27
- data/spec/fluq/feed/tsv_spec.rb +0 -27
- data/spec/fluq/input/socket/connection_spec.rb +0 -35
- data/spec/fluq/mixins/logger_spec.rb +0 -25
- data/spec/fluq/reactor_spec.rb +0 -69
- data/spec/scenario/config/nested/common.rb +0 -3
data/lib/fluq/input/base.rb
CHANGED
@@ -1,59 +1,63 @@
|
|
1
1
|
class FluQ::Input::Base
|
2
|
+
include Celluloid::IO
|
2
3
|
include FluQ::Mixins::Loggable
|
3
|
-
|
4
|
-
# @attr_reader [FluQ::Reactor] reactor reference
|
5
|
-
attr_reader :reactor
|
4
|
+
finalizer :before_terminate
|
6
5
|
|
7
6
|
# @attr_reader [Hash] config
|
8
7
|
attr_reader :config
|
9
8
|
|
10
|
-
# @
|
9
|
+
# @attr_reader [FluQ::Worker] worker
|
10
|
+
attr_reader :worker
|
11
|
+
|
12
|
+
# @param [String] source feed name
|
13
|
+
# @param [Array<Class,multiple>] handlers handler builders
|
11
14
|
# @param [Hash] options various configuration options
|
12
|
-
def initialize(
|
15
|
+
def initialize(source, handlers, options = {})
|
13
16
|
super()
|
14
|
-
@
|
15
|
-
|
17
|
+
@config = defaults.merge(options)
|
18
|
+
configure
|
19
|
+
|
20
|
+
@worker = FluQ::Worker.new_link [source, name].join(":"), handlers
|
21
|
+
logger.info "#{source}: listening to #{description}"
|
22
|
+
async.run
|
16
23
|
end
|
17
24
|
|
18
|
-
# @return [String]
|
25
|
+
# @return [String] short name
|
19
26
|
def name
|
20
27
|
@name ||= self.class.name.split("::")[-1].downcase
|
21
28
|
end
|
22
29
|
|
23
|
-
#
|
24
|
-
def
|
30
|
+
# @return [String] descriptive name
|
31
|
+
def description
|
32
|
+
name
|
25
33
|
end
|
26
34
|
|
27
|
-
#
|
28
|
-
|
29
|
-
def new_buffer
|
30
|
-
buffer_klass.new config[:buffer_options]
|
35
|
+
# Start the input
|
36
|
+
def run
|
31
37
|
end
|
32
38
|
|
33
|
-
#
|
34
|
-
# @param [
|
35
|
-
def
|
36
|
-
|
37
|
-
reactor.process(events)
|
38
|
-
end
|
39
|
-
rescue => ex
|
40
|
-
logger.crash "#{self.class.name} failure: #{ex.message} (#{ex.class.name})", ex
|
41
|
-
ensure
|
42
|
-
buffer.close if buffer
|
39
|
+
# Processes data
|
40
|
+
# @param [String] data
|
41
|
+
def process(data)
|
42
|
+
worker.process format.parse(data)
|
43
43
|
end
|
44
44
|
|
45
45
|
protected
|
46
46
|
|
47
|
-
|
48
|
-
|
47
|
+
# @abstract callback for configuration initialization
|
48
|
+
def configure
|
49
|
+
end
|
50
|
+
|
51
|
+
# @abstract callback before termination
|
52
|
+
def before_terminate
|
49
53
|
end
|
50
54
|
|
51
|
-
def
|
52
|
-
@
|
55
|
+
def format
|
56
|
+
@format ||= FluQ::Format.const_get(config[:format].to_s.capitalize).new(config[:format_options])
|
53
57
|
end
|
54
58
|
|
55
59
|
def defaults
|
56
|
-
{
|
60
|
+
{ format: "json", format_options: {} }
|
57
61
|
end
|
58
62
|
|
59
63
|
end
|
data/lib/fluq/input/socket.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
class FluQ::Input::Socket < FluQ::Input::Base
|
2
|
+
MAXLEN = 16 * 1024
|
2
3
|
|
3
4
|
# @attr_reader [URI] url the URL
|
4
5
|
attr_reader :url
|
@@ -9,42 +10,71 @@ class FluQ::Input::Socket < FluQ::Input::Base
|
|
9
10
|
# @raises [URI::InvalidURIError] if invalid URL is given
|
10
11
|
# @example Launch a server
|
11
12
|
#
|
12
|
-
# server = FluQ::Server.new(
|
13
|
+
# server = FluQ::Server.new(runner, bind: "tcp://localhost:7654")
|
13
14
|
#
|
14
15
|
def initialize(*)
|
15
16
|
super
|
17
|
+
end
|
16
18
|
|
17
|
-
|
18
|
-
|
19
|
+
# @return [String] short name
|
20
|
+
def name
|
21
|
+
@url ? @url.scheme : super
|
19
22
|
end
|
20
23
|
|
21
24
|
# @return [String] descriptive name
|
22
|
-
def
|
23
|
-
|
25
|
+
def description
|
26
|
+
"socket (#{@url})"
|
24
27
|
end
|
25
28
|
|
26
29
|
# Start the server
|
27
30
|
def run
|
28
|
-
|
29
|
-
|
31
|
+
super
|
32
|
+
|
33
|
+
@server = case @url.scheme
|
30
34
|
when 'tcp'
|
31
|
-
|
35
|
+
TCPServer.new(@url.host, @url.port)
|
32
36
|
when 'udp'
|
33
|
-
|
37
|
+
UDPSocket.new.tap {|s| s.bind(@url.host, @url.port) }
|
34
38
|
when 'unix'
|
35
|
-
|
39
|
+
UNIXServer.open(@url.path)
|
36
40
|
end
|
41
|
+
|
42
|
+
case @url.scheme
|
43
|
+
when 'udp'
|
44
|
+
loop { process @server.recvfrom(MAXLEN)[0] }
|
45
|
+
else
|
46
|
+
loop { async.handle_connection @server.accept }
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
# @return [Boolean] true when listening
|
51
|
+
def listening?
|
52
|
+
!!@server
|
37
53
|
end
|
38
54
|
|
39
55
|
protected
|
40
56
|
|
41
|
-
# @
|
42
|
-
def
|
43
|
-
|
57
|
+
# @abstract callback for configuration initialization
|
58
|
+
def configure
|
59
|
+
raise ArgumentError, 'No URL to bind to provided, make sure you pass :bind option' unless config[:bind]
|
60
|
+
@url = FluQ::URL.parse config[:bind], %w|tcp udp unix|
|
44
61
|
end
|
45
62
|
|
46
|
-
|
63
|
+
# Handle a single connection
|
64
|
+
def handle_connection(socket)
|
65
|
+
loop do
|
66
|
+
process socket.readpartial(MAXLEN)
|
67
|
+
end
|
68
|
+
rescue EOFError
|
69
|
+
ensure
|
70
|
+
socket.close
|
71
|
+
end
|
72
|
+
|
73
|
+
def before_terminate
|
74
|
+
return unless @server
|
75
|
+
|
76
|
+
@server.close
|
77
|
+
File.delete @url.path if @url.scheme == "unix"
|
78
|
+
end
|
47
79
|
|
48
|
-
%w'connection'.each do |name|
|
49
|
-
require "fluq/input/socket/#{name}"
|
50
80
|
end
|
data/lib/fluq/mixins.rb
CHANGED
data/lib/fluq/runner.rb
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
class FluQ::Runner
|
2
|
+
include FluQ::Mixins::Loggable
|
3
|
+
|
4
|
+
# Runs the runner (blocking)
|
5
|
+
def self.run(&block)
|
6
|
+
new(&block).run
|
7
|
+
end
|
8
|
+
|
9
|
+
# Constructor
|
10
|
+
def initialize(&block)
|
11
|
+
@feeds = Celluloid::SupervisionGroup.new
|
12
|
+
block.call(self) if block
|
13
|
+
end
|
14
|
+
|
15
|
+
# @return [Array<FluQ::Feed>]
|
16
|
+
def feeds
|
17
|
+
@feeds.actors
|
18
|
+
end
|
19
|
+
|
20
|
+
# Registers a new feed
|
21
|
+
# @param [String] name
|
22
|
+
def feed(name, &block)
|
23
|
+
@feeds.supervise FluQ::Feed, name, &block
|
24
|
+
end
|
25
|
+
|
26
|
+
# Starts the runner, blocking
|
27
|
+
def run
|
28
|
+
loop { sleep 5 while @feeds.alive? }
|
29
|
+
end
|
30
|
+
|
31
|
+
# Terminates the runner
|
32
|
+
def terminate
|
33
|
+
@feeds.terminate
|
34
|
+
end
|
35
|
+
|
36
|
+
# @return [String] introspection
|
37
|
+
def inspect
|
38
|
+
"#<#{self.class.name} feeds: #{feeds.map(&:name).inspect}>"
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
data/lib/fluq/testing.rb
CHANGED
@@ -2,7 +2,6 @@ require 'fluq'
|
|
2
2
|
|
3
3
|
module FluQ::Testing
|
4
4
|
extend self
|
5
|
-
EXCEPTION_TRACKER = ->ex { FluQ::Testing.exceptions.push(ex) }
|
6
5
|
|
7
6
|
def wait_until(opts = {}, &block)
|
8
7
|
tick = opts[:tick] || 0.01
|
@@ -11,15 +10,6 @@ module FluQ::Testing
|
|
11
10
|
rescue Timeout::Error
|
12
11
|
end
|
13
12
|
|
14
|
-
def exceptions
|
15
|
-
@exceptions ||= []
|
16
|
-
end
|
17
|
-
|
18
|
-
def track_exceptions!(logger = FluQ.logger)
|
19
|
-
return if logger.exception_handlers.include?(EXCEPTION_TRACKER)
|
20
|
-
logger.exception_handler(&EXCEPTION_TRACKER)
|
21
|
-
end
|
22
|
-
|
23
13
|
end
|
24
14
|
|
25
15
|
class FluQ::Handler::Test < FluQ::Handler::Base
|
@@ -30,8 +20,12 @@ class FluQ::Handler::Test < FluQ::Handler::Base
|
|
30
20
|
@events = []
|
31
21
|
end
|
32
22
|
|
23
|
+
def filter(events)
|
24
|
+
events.reject {|e| e.key?("filter") }
|
25
|
+
end
|
26
|
+
|
33
27
|
def on_events(events)
|
34
|
-
raise RuntimeError, "Test Failure!" if events.any? {|e| e.
|
28
|
+
raise RuntimeError, "Test Failure!" if events.any? {|e| e.key?("error") }
|
35
29
|
@events.concat events
|
36
30
|
end
|
37
31
|
|
data/lib/fluq/version.rb
CHANGED
data/lib/fluq/worker.rb
ADDED
@@ -0,0 +1,73 @@
|
|
1
|
+
class FluQ::Worker
|
2
|
+
include Celluloid
|
3
|
+
include FluQ::Mixins::Loggable
|
4
|
+
finalizer :finalize
|
5
|
+
|
6
|
+
attr_reader :prefix, :handlers
|
7
|
+
|
8
|
+
# @param [Array<Class,Array>] handlers handler builders
|
9
|
+
def initialize(prefix, handlers = [])
|
10
|
+
@prefix = prefix
|
11
|
+
@handlers = []
|
12
|
+
@observer = observe
|
13
|
+
|
14
|
+
handlers.each do |klass, *args|
|
15
|
+
add klass, *args
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
# @param [Array<FluQ::Event>] events to process
|
20
|
+
def process(events)
|
21
|
+
events.freeze # Freeze events, don't allow individual handlers to modify them
|
22
|
+
handlers.each do |handler|
|
23
|
+
on_events(handler, Time.now, events)
|
24
|
+
end
|
25
|
+
true
|
26
|
+
end
|
27
|
+
|
28
|
+
# Adds a handler
|
29
|
+
# @param [Class<FluQ::Handler::Base>] klass handler class
|
30
|
+
# @param [multiple] args handler initialize arguments
|
31
|
+
def add(klass, *args)
|
32
|
+
handler = klass.new(*args)
|
33
|
+
handlers.push handler
|
34
|
+
handler
|
35
|
+
end
|
36
|
+
|
37
|
+
protected
|
38
|
+
|
39
|
+
def finalize
|
40
|
+
@observer.kill if @observer
|
41
|
+
end
|
42
|
+
|
43
|
+
def on_events(handler, start, events)
|
44
|
+
matching = handler.filter(events)
|
45
|
+
::Timeout::timeout handler.config[:timeout] do
|
46
|
+
handler.on_events(matching)
|
47
|
+
logger.info { "#{prefix}:#{handler.name} #{matching.size}/#{events.size} events in #{((Time.now - start) * 1000).round}ms" }
|
48
|
+
end unless matching.empty?
|
49
|
+
end
|
50
|
+
|
51
|
+
private
|
52
|
+
|
53
|
+
def next_timers
|
54
|
+
handlers.map do |h|
|
55
|
+
h.timers unless h.timers.empty?
|
56
|
+
end.compact.sort_by(&:wait_interval)[0]
|
57
|
+
end
|
58
|
+
|
59
|
+
def observe
|
60
|
+
parent = Thread.current
|
61
|
+
Thread.new do
|
62
|
+
loop do
|
63
|
+
begin
|
64
|
+
timers = next_timers
|
65
|
+
timers ? timers.wait : sleep(0.1)
|
66
|
+
rescue => e
|
67
|
+
parent.raise(e)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe FluQ::DSL::Feed do
|
4
|
+
|
5
|
+
let(:root) { FluQ::DSL::Root.new FluQ.root.join('../scenario/config/test.rb') }
|
6
|
+
subject { root.feeds.first }
|
7
|
+
|
8
|
+
it { should be_instance_of(described_class) }
|
9
|
+
it { should have(1).inputs }
|
10
|
+
it { should have(1).handlers }
|
11
|
+
|
12
|
+
it 'should find & configure input' do
|
13
|
+
subject.input(:socket) do
|
14
|
+
bind 'tcp://localhost:76543'
|
15
|
+
end
|
16
|
+
subject.should have(2).inputs
|
17
|
+
subject.inputs.last.should == [FluQ::Input::Socket, {bind: "tcp://localhost:76543"}]
|
18
|
+
end
|
19
|
+
|
20
|
+
it 'should find & configure handler' do
|
21
|
+
subject.handler(:log)
|
22
|
+
subject.should have(2).handlers
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'should find namespaced handler' do
|
26
|
+
subject.handler(:custom, :test_handler) do
|
27
|
+
to 'tcp://localhost:87654'
|
28
|
+
end
|
29
|
+
subject.should have(2).handlers
|
30
|
+
subject.handlers.last.should == [FluQ::Handler::Custom::TestHandler, {to: "tcp://localhost:87654"}]
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe FluQ::DSL::Root do
|
4
|
+
|
5
|
+
let(:runner) { FluQ::Runner.new }
|
6
|
+
subject { described_class.new FluQ.root.join('../scenario/config/test.rb').to_s }
|
7
|
+
|
8
|
+
its(:feeds) { should have(2).items }
|
9
|
+
its("feeds.first") { should be_instance_of(FluQ::DSL::Feed) }
|
10
|
+
|
11
|
+
it 'should apply configuration' do
|
12
|
+
subject.apply(runner)
|
13
|
+
runner.should have(2).feeds
|
14
|
+
|
15
|
+
feed = runner.feeds.first
|
16
|
+
feed.should have(1).inputs
|
17
|
+
feed.should have(1).handlers
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
data/spec/fluq/event_spec.rb
CHANGED
@@ -2,24 +2,29 @@ require 'spec_helper'
|
|
2
2
|
|
3
3
|
describe FluQ::Event do
|
4
4
|
|
5
|
-
subject { described_class.new
|
5
|
+
subject { described_class.new({"a" => "v1", "b" => "v2"}, "1313131313") }
|
6
6
|
|
7
7
|
it { should be_a(Hash) }
|
8
|
-
its(:
|
8
|
+
its(:meta) { should == {} }
|
9
9
|
its(:timestamp) { should == 1313131313 }
|
10
10
|
its(:time) { should be_instance_of(Time) }
|
11
11
|
its(:time) { should be_utc }
|
12
|
-
its(:to_a) { should == ["some.tag", 1313131313, "a" => "v1", "b" => "v2"] }
|
13
|
-
its(:to_tsv) { should == %(some.tag\t1313131313\t{"a":"v1","b":"v2"}) }
|
14
|
-
its(:to_json) { should == %({"a":"v1","b":"v2","=":"some.tag","@":1313131313}) }
|
15
|
-
its(:to_msgpack) { should == "\x84\xA1a\xA2v1\xA1b\xA2v2\xA1=\xA8some.tag\xA1@\xCEND\xCB1".force_encoding(Encoding::BINARY) }
|
16
|
-
its(:inspect) { should == %(["some.tag", 1313131313, {"a"=>"v1", "b"=>"v2"}]) }
|
17
12
|
|
18
13
|
it "should be comparable" do
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
14
|
+
other = described_class.new({"a" => "v1", "b" => "v2"}, "1313131313")
|
15
|
+
|
16
|
+
subject.should == other
|
17
|
+
other.meta[:some] = "thing"
|
18
|
+
subject.should == other
|
19
|
+
other["c"] = "d"
|
20
|
+
subject.should_not == other
|
21
|
+
subject.should_not == described_class.new({"a" => "v1", "b" => "v2"}, "1313131312")
|
22
|
+
end
|
23
|
+
|
24
|
+
it "should be inspectable" do
|
25
|
+
subject.inspect.should == %(#<FluQ::Event(1313131313) data:{"a"=>"v1", "b"=>"v2"} meta:{}>)
|
26
|
+
subject.meta[:some] = "thing"
|
27
|
+
subject.inspect.should == %(#<FluQ::Event(1313131313) data:{"a"=>"v1", "b"=>"v2"} meta:{:some=>"thing"}>)
|
23
28
|
end
|
24
29
|
|
25
|
-
end
|
30
|
+
end
|