fluq 0.7.5 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.travis.yml +3 -0
  4. data/Gemfile +12 -1
  5. data/Gemfile.lock +44 -8
  6. data/README.md +24 -6
  7. data/Rakefile +8 -1
  8. data/benchmark/socket.rb +13 -25
  9. data/examples/config/multi.rb +52 -0
  10. data/examples/config/simple.rb +15 -0
  11. data/fluq.gemspec +3 -3
  12. data/lib/fluq.rb +22 -16
  13. data/lib/fluq/cli.rb +3 -12
  14. data/lib/fluq/dsl.rb +2 -45
  15. data/lib/fluq/dsl/base.rb +11 -0
  16. data/lib/fluq/dsl/feed.rb +24 -0
  17. data/lib/fluq/dsl/root.rb +35 -0
  18. data/lib/fluq/event.rb +9 -28
  19. data/lib/fluq/feed.rb +40 -5
  20. data/lib/fluq/format.rb +6 -0
  21. data/lib/fluq/format/base.rb +42 -0
  22. data/lib/fluq/format/json.rb +17 -0
  23. data/lib/fluq/format/lines.rb +27 -0
  24. data/lib/fluq/format/msgpack.rb +28 -0
  25. data/lib/fluq/format/tsv.rb +19 -0
  26. data/lib/fluq/handler.rb +1 -1
  27. data/lib/fluq/handler/base.rb +11 -38
  28. data/lib/fluq/handler/log.rb +12 -14
  29. data/lib/fluq/handler/noop.rb +2 -0
  30. data/lib/fluq/input/base.rb +33 -29
  31. data/lib/fluq/input/socket.rb +46 -16
  32. data/lib/fluq/mixins.rb +2 -2
  33. data/lib/fluq/runner.rb +41 -0
  34. data/lib/fluq/testing.rb +5 -11
  35. data/lib/fluq/version.rb +1 -1
  36. data/lib/fluq/worker.rb +73 -0
  37. data/spec/fluq/dsl/feed_spec.rb +33 -0
  38. data/spec/fluq/dsl/root_spec.rb +20 -0
  39. data/spec/fluq/event_spec.rb +17 -12
  40. data/spec/fluq/feed_spec.rb +24 -0
  41. data/spec/fluq/format/base_spec.rb +9 -0
  42. data/spec/fluq/format/json_spec.rb +22 -0
  43. data/spec/fluq/format/lines_spec.rb +20 -0
  44. data/spec/fluq/format/msgpack_spec.rb +22 -0
  45. data/spec/fluq/format/tsv_spec.rb +21 -0
  46. data/spec/fluq/handler/base_spec.rb +7 -52
  47. data/spec/fluq/handler/log_spec.rb +11 -14
  48. data/spec/fluq/handler/{null_spec.rb → noop_spec.rb} +1 -3
  49. data/spec/fluq/input/base_spec.rb +48 -15
  50. data/spec/fluq/input/socket_spec.rb +34 -26
  51. data/spec/fluq/mixins/loggable_spec.rb +2 -2
  52. data/spec/fluq/runner_spec.rb +18 -0
  53. data/spec/fluq/worker_spec.rb +87 -0
  54. data/spec/fluq_spec.rb +1 -2
  55. data/spec/scenario/config/nested/feed1.rb +6 -0
  56. data/spec/scenario/config/test.rb +8 -2
  57. data/spec/spec_helper.rb +7 -26
  58. metadata +62 -62
  59. data/benchmark/logging.rb +0 -37
  60. data/examples/common.rb +0 -3
  61. data/examples/simple.rb +0 -5
  62. data/lib/fluq/buffer.rb +0 -6
  63. data/lib/fluq/buffer/base.rb +0 -51
  64. data/lib/fluq/buffer/file.rb +0 -68
  65. data/lib/fluq/feed/base.rb +0 -37
  66. data/lib/fluq/feed/json.rb +0 -28
  67. data/lib/fluq/feed/msgpack.rb +0 -27
  68. data/lib/fluq/feed/tsv.rb +0 -30
  69. data/lib/fluq/handler/null.rb +0 -4
  70. data/lib/fluq/input/socket/connection.rb +0 -41
  71. data/lib/fluq/mixins/logger.rb +0 -26
  72. data/lib/fluq/reactor.rb +0 -79
  73. data/spec/fluq/buffer/base_spec.rb +0 -21
  74. data/spec/fluq/buffer/file_spec.rb +0 -47
  75. data/spec/fluq/dsl_spec.rb +0 -43
  76. data/spec/fluq/feed/base_spec.rb +0 -15
  77. data/spec/fluq/feed/json_spec.rb +0 -27
  78. data/spec/fluq/feed/msgpack_spec.rb +0 -27
  79. data/spec/fluq/feed/tsv_spec.rb +0 -27
  80. data/spec/fluq/input/socket/connection_spec.rb +0 -35
  81. data/spec/fluq/mixins/logger_spec.rb +0 -25
  82. data/spec/fluq/reactor_spec.rb +0 -69
  83. data/spec/scenario/config/nested/common.rb +0 -3
@@ -0,0 +1,2 @@
1
+ class FluQ::Handler::Noop < FluQ::Handler::Base
2
+ end
@@ -1,59 +1,63 @@
1
1
  class FluQ::Input::Base
2
+ include Celluloid::IO
2
3
  include FluQ::Mixins::Loggable
3
-
4
- # @attr_reader [FluQ::Reactor] reactor reference
5
- attr_reader :reactor
4
+ finalizer :before_terminate
6
5
 
7
6
  # @attr_reader [Hash] config
8
7
  attr_reader :config
9
8
 
10
- # @param [FluQ::Reactor] reactor
9
+ # @attr_reader [FluQ::Worker] worker
10
+ attr_reader :worker
11
+
12
+ # @param [String] source feed name
13
+ # @param [Array<Class,multiple>] handlers handler builders
11
14
  # @param [Hash] options various configuration options
12
- def initialize(reactor, options = {})
15
+ def initialize(source, handlers, options = {})
13
16
  super()
14
- @reactor = reactor
15
- @config = defaults.merge(options)
17
+ @config = defaults.merge(options)
18
+ configure
19
+
20
+ @worker = FluQ::Worker.new_link [source, name].join(":"), handlers
21
+ logger.info "#{source}: listening to #{description}"
22
+ async.run
16
23
  end
17
24
 
18
- # @return [String] descriptive name
25
+ # @return [String] short name
19
26
  def name
20
27
  @name ||= self.class.name.split("::")[-1].downcase
21
28
  end
22
29
 
23
- # Start the input
24
- def run
30
+ # @return [String] descriptive name
31
+ def description
32
+ name
25
33
  end
26
34
 
27
- # Creates a new buffer object
28
- # @return [FluQ::Buffer::Base] a new buffer
29
- def new_buffer
30
- buffer_klass.new config[:buffer_options]
35
+ # Start the input
36
+ def run
31
37
  end
32
38
 
33
- # Flushes and closes a buffer
34
- # @param [FluQ::Buffer::Base] buffer
35
- def flush!(buffer)
36
- feed_klass.new(buffer).each_slice(10_000) do |events|
37
- reactor.process(events)
38
- end
39
- rescue => ex
40
- logger.crash "#{self.class.name} failure: #{ex.message} (#{ex.class.name})", ex
41
- ensure
42
- buffer.close if buffer
39
+ # Processes data
40
+ # @param [String] data
41
+ def process(data)
42
+ worker.process format.parse(data)
43
43
  end
44
44
 
45
45
  protected
46
46
 
47
- def buffer_klass
48
- @buffer_klass ||= FluQ::Buffer.const_get(config[:buffer].to_s.capitalize)
47
+ # @abstract callback for configuration initialization
48
+ def configure
49
+ end
50
+
51
+ # @abstract callback before termination
52
+ def before_terminate
49
53
  end
50
54
 
51
- def feed_klass
52
- @feed_klass ||= FluQ::Feed.const_get(config[:feed].to_s.capitalize)
55
+ def format
56
+ @format ||= FluQ::Format.const_get(config[:format].to_s.capitalize).new(config[:format_options])
53
57
  end
54
58
 
55
59
  def defaults
56
- { buffer: "file", feed: "msgpack", buffer_options: {} }
60
+ { format: "json", format_options: {} }
57
61
  end
58
62
 
59
63
  end
@@ -1,4 +1,5 @@
1
1
  class FluQ::Input::Socket < FluQ::Input::Base
2
+ MAXLEN = 16 * 1024
2
3
 
3
4
  # @attr_reader [URI] url the URL
4
5
  attr_reader :url
@@ -9,42 +10,71 @@ class FluQ::Input::Socket < FluQ::Input::Base
9
10
  # @raises [URI::InvalidURIError] if invalid URL is given
10
11
  # @example Launch a server
11
12
  #
12
- # server = FluQ::Server.new(reactor, bind: "tcp://localhost:7654")
13
+ # server = FluQ::Server.new(runner, bind: "tcp://localhost:7654")
13
14
  #
14
15
  def initialize(*)
15
16
  super
17
+ end
16
18
 
17
- raise ArgumentError, 'No URL to bind to provided, make sure you pass :bind option' unless config[:bind]
18
- @url = FluQ::URL.parse(config[:bind], protocols)
19
+ # @return [String] short name
20
+ def name
21
+ @url ? @url.scheme : super
19
22
  end
20
23
 
21
24
  # @return [String] descriptive name
22
- def name
23
- @name ||= "#{super} (#{@url})"
25
+ def description
26
+ "socket (#{@url})"
24
27
  end
25
28
 
26
29
  # Start the server
27
30
  def run
28
- args = [self.class::Connection, self]
29
- case @url.scheme
31
+ super
32
+
33
+ @server = case @url.scheme
30
34
  when 'tcp'
31
- EventMachine.start_server @url.host, @url.port, *args
35
+ TCPServer.new(@url.host, @url.port)
32
36
  when 'udp'
33
- EventMachine.open_datagram_socket @url.host, @url.port, *args
37
+ UDPSocket.new.tap {|s| s.bind(@url.host, @url.port) }
34
38
  when 'unix'
35
- EventMachine.start_server @url.path, *args
39
+ UNIXServer.open(@url.path)
36
40
  end
41
+
42
+ case @url.scheme
43
+ when 'udp'
44
+ loop { process @server.recvfrom(MAXLEN)[0] }
45
+ else
46
+ loop { async.handle_connection @server.accept }
47
+ end
48
+ end
49
+
50
+ # @return [Boolean] true when listening
51
+ def listening?
52
+ !!@server
37
53
  end
38
54
 
39
55
  protected
40
56
 
41
- # @return [Array] supported protocols
42
- def protocols
43
- ["tcp", "udp", "unix"]
57
+ # @abstract callback for configuration initialization
58
+ def configure
59
+ raise ArgumentError, 'No URL to bind to provided, make sure you pass :bind option' unless config[:bind]
60
+ @url = FluQ::URL.parse config[:bind], %w|tcp udp unix|
44
61
  end
45
62
 
46
- end
63
+ # Handle a single connection
64
+ def handle_connection(socket)
65
+ loop do
66
+ process socket.readpartial(MAXLEN)
67
+ end
68
+ rescue EOFError
69
+ ensure
70
+ socket.close
71
+ end
72
+
73
+ def before_terminate
74
+ return unless @server
75
+
76
+ @server.close
77
+ File.delete @url.path if @url.scheme == "unix"
78
+ end
47
79
 
48
- %w'connection'.each do |name|
49
- require "fluq/input/socket/#{name}"
50
80
  end
data/lib/fluq/mixins.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  module FluQ::Mixins
2
2
  end
3
3
 
4
- %w'loggable logger'.each do |name|
4
+ %w'loggable'.each do |name|
5
5
  require "fluq/mixins/#{name}"
6
- end
6
+ end
@@ -0,0 +1,41 @@
1
+ class FluQ::Runner
2
+ include FluQ::Mixins::Loggable
3
+
4
+ # Runs the runner (blocking)
5
+ def self.run(&block)
6
+ new(&block).run
7
+ end
8
+
9
+ # Constructor
10
+ def initialize(&block)
11
+ @feeds = Celluloid::SupervisionGroup.new
12
+ block.call(self) if block
13
+ end
14
+
15
+ # @return [Array<FluQ::Feed>]
16
+ def feeds
17
+ @feeds.actors
18
+ end
19
+
20
+ # Registers a new feed
21
+ # @param [String] name
22
+ def feed(name, &block)
23
+ @feeds.supervise FluQ::Feed, name, &block
24
+ end
25
+
26
+ # Starts the runner, blocking
27
+ def run
28
+ loop { sleep 5 while @feeds.alive? }
29
+ end
30
+
31
+ # Terminates the runner
32
+ def terminate
33
+ @feeds.terminate
34
+ end
35
+
36
+ # @return [String] introspection
37
+ def inspect
38
+ "#<#{self.class.name} feeds: #{feeds.map(&:name).inspect}>"
39
+ end
40
+
41
+ end
data/lib/fluq/testing.rb CHANGED
@@ -2,7 +2,6 @@ require 'fluq'
2
2
 
3
3
  module FluQ::Testing
4
4
  extend self
5
- EXCEPTION_TRACKER = ->ex { FluQ::Testing.exceptions.push(ex) }
6
5
 
7
6
  def wait_until(opts = {}, &block)
8
7
  tick = opts[:tick] || 0.01
@@ -11,15 +10,6 @@ module FluQ::Testing
11
10
  rescue Timeout::Error
12
11
  end
13
12
 
14
- def exceptions
15
- @exceptions ||= []
16
- end
17
-
18
- def track_exceptions!(logger = FluQ.logger)
19
- return if logger.exception_handlers.include?(EXCEPTION_TRACKER)
20
- logger.exception_handler(&EXCEPTION_TRACKER)
21
- end
22
-
23
13
  end
24
14
 
25
15
  class FluQ::Handler::Test < FluQ::Handler::Base
@@ -30,8 +20,12 @@ class FluQ::Handler::Test < FluQ::Handler::Base
30
20
  @events = []
31
21
  end
32
22
 
23
+ def filter(events)
24
+ events.reject {|e| e.key?("filter") }
25
+ end
26
+
33
27
  def on_events(events)
34
- raise RuntimeError, "Test Failure!" if events.any? {|e| e.tag == "error.event" }
28
+ raise RuntimeError, "Test Failure!" if events.any? {|e| e.key?("error") }
35
29
  @events.concat events
36
30
  end
37
31
 
data/lib/fluq/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module FluQ
2
- VERSION = "0.7.5"
2
+ VERSION = "0.8.0"
3
3
  end
@@ -0,0 +1,73 @@
1
+ class FluQ::Worker
2
+ include Celluloid
3
+ include FluQ::Mixins::Loggable
4
+ finalizer :finalize
5
+
6
+ attr_reader :prefix, :handlers
7
+
8
+ # @param [Array<Class,Array>] handlers handler builders
9
+ def initialize(prefix, handlers = [])
10
+ @prefix = prefix
11
+ @handlers = []
12
+ @observer = observe
13
+
14
+ handlers.each do |klass, *args|
15
+ add klass, *args
16
+ end
17
+ end
18
+
19
+ # @param [Array<FluQ::Event>] events to process
20
+ def process(events)
21
+ events.freeze # Freeze events, don't allow individual handlers to modify them
22
+ handlers.each do |handler|
23
+ on_events(handler, Time.now, events)
24
+ end
25
+ true
26
+ end
27
+
28
+ # Adds a handler
29
+ # @param [Class<FluQ::Handler::Base>] klass handler class
30
+ # @param [multiple] args handler initialize arguments
31
+ def add(klass, *args)
32
+ handler = klass.new(*args)
33
+ handlers.push handler
34
+ handler
35
+ end
36
+
37
+ protected
38
+
39
+ def finalize
40
+ @observer.kill if @observer
41
+ end
42
+
43
+ def on_events(handler, start, events)
44
+ matching = handler.filter(events)
45
+ ::Timeout::timeout handler.config[:timeout] do
46
+ handler.on_events(matching)
47
+ logger.info { "#{prefix}:#{handler.name} #{matching.size}/#{events.size} events in #{((Time.now - start) * 1000).round}ms" }
48
+ end unless matching.empty?
49
+ end
50
+
51
+ private
52
+
53
+ def next_timers
54
+ handlers.map do |h|
55
+ h.timers unless h.timers.empty?
56
+ end.compact.sort_by(&:wait_interval)[0]
57
+ end
58
+
59
+ def observe
60
+ parent = Thread.current
61
+ Thread.new do
62
+ loop do
63
+ begin
64
+ timers = next_timers
65
+ timers ? timers.wait : sleep(0.1)
66
+ rescue => e
67
+ parent.raise(e)
68
+ end
69
+ end
70
+ end
71
+ end
72
+
73
+ end
@@ -0,0 +1,33 @@
1
+ require 'spec_helper'
2
+
3
+ describe FluQ::DSL::Feed do
4
+
5
+ let(:root) { FluQ::DSL::Root.new FluQ.root.join('../scenario/config/test.rb') }
6
+ subject { root.feeds.first }
7
+
8
+ it { should be_instance_of(described_class) }
9
+ it { should have(1).inputs }
10
+ it { should have(1).handlers }
11
+
12
+ it 'should find & configure input' do
13
+ subject.input(:socket) do
14
+ bind 'tcp://localhost:76543'
15
+ end
16
+ subject.should have(2).inputs
17
+ subject.inputs.last.should == [FluQ::Input::Socket, {bind: "tcp://localhost:76543"}]
18
+ end
19
+
20
+ it 'should find & configure handler' do
21
+ subject.handler(:log)
22
+ subject.should have(2).handlers
23
+ end
24
+
25
+ it 'should find namespaced handler' do
26
+ subject.handler(:custom, :test_handler) do
27
+ to 'tcp://localhost:87654'
28
+ end
29
+ subject.should have(2).handlers
30
+ subject.handlers.last.should == [FluQ::Handler::Custom::TestHandler, {to: "tcp://localhost:87654"}]
31
+ end
32
+
33
+ end
@@ -0,0 +1,20 @@
1
+ require 'spec_helper'
2
+
3
+ describe FluQ::DSL::Root do
4
+
5
+ let(:runner) { FluQ::Runner.new }
6
+ subject { described_class.new FluQ.root.join('../scenario/config/test.rb').to_s }
7
+
8
+ its(:feeds) { should have(2).items }
9
+ its("feeds.first") { should be_instance_of(FluQ::DSL::Feed) }
10
+
11
+ it 'should apply configuration' do
12
+ subject.apply(runner)
13
+ runner.should have(2).feeds
14
+
15
+ feed = runner.feeds.first
16
+ feed.should have(1).inputs
17
+ feed.should have(1).handlers
18
+ end
19
+
20
+ end
@@ -2,24 +2,29 @@ require 'spec_helper'
2
2
 
3
3
  describe FluQ::Event do
4
4
 
5
- subject { described_class.new :"some.tag", "1313131313", "a" => "v1", "b" => "v2" }
5
+ subject { described_class.new({"a" => "v1", "b" => "v2"}, "1313131313") }
6
6
 
7
7
  it { should be_a(Hash) }
8
- its(:tag) { should == "some.tag" }
8
+ its(:meta) { should == {} }
9
9
  its(:timestamp) { should == 1313131313 }
10
10
  its(:time) { should be_instance_of(Time) }
11
11
  its(:time) { should be_utc }
12
- its(:to_a) { should == ["some.tag", 1313131313, "a" => "v1", "b" => "v2"] }
13
- its(:to_tsv) { should == %(some.tag\t1313131313\t{"a":"v1","b":"v2"}) }
14
- its(:to_json) { should == %({"a":"v1","b":"v2","=":"some.tag","@":1313131313}) }
15
- its(:to_msgpack) { should == "\x84\xA1a\xA2v1\xA1b\xA2v2\xA1=\xA8some.tag\xA1@\xCEND\xCB1".force_encoding(Encoding::BINARY) }
16
- its(:inspect) { should == %(["some.tag", 1313131313, {"a"=>"v1", "b"=>"v2"}]) }
17
12
 
18
13
  it "should be comparable" do
19
- subject.should == { "a" => "v1", "b" => "v2" }
20
- subject.should == ["some.tag", 1313131313, "a" => "v1", "b" => "v2"]
21
- [subject].should == [{ "a" => "v1", "b" => "v2" }]
22
- [subject, subject].should == [["some.tag", 1313131313, "a" => "v1", "b" => "v2"]] * 2
14
+ other = described_class.new({"a" => "v1", "b" => "v2"}, "1313131313")
15
+
16
+ subject.should == other
17
+ other.meta[:some] = "thing"
18
+ subject.should == other
19
+ other["c"] = "d"
20
+ subject.should_not == other
21
+ subject.should_not == described_class.new({"a" => "v1", "b" => "v2"}, "1313131312")
22
+ end
23
+
24
+ it "should be inspectable" do
25
+ subject.inspect.should == %(#<FluQ::Event(1313131313) data:{"a"=>"v1", "b"=>"v2"} meta:{}>)
26
+ subject.meta[:some] = "thing"
27
+ subject.inspect.should == %(#<FluQ::Event(1313131313) data:{"a"=>"v1", "b"=>"v2"} meta:{:some=>"thing"}>)
23
28
  end
24
29
 
25
- end
30
+ end