fwd 0.3.2 → 0.3.3

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fwd (0.3.2)
+    fwd (0.3.3)
       connection_pool
       eventmachine-le
       servolux

data/benchmark/performance.rb

@@ -6,29 +6,31 @@ require 'socket'
 require 'fileutils'
 
 root = Pathname.new(File.expand_path('../..', __FILE__))
-tmp  = root.join("tmp/benchmark")
-FileUtils.rm_rf tmp
-FileUtils.mkdir_p tmp
+TMP  = root.join("tmp/benchmark")
+OUT  = TMP.join('out.txt')
 
-OUT = tmp.join('out.txt')
-FWD = fork { exec "#{root}/bin/fwd-rb --flush 10000:2 -F tcp://0.0.0.0:7291 --path #{tmp} -v" }
-NCC = fork { exec "nc -vlp 7291 > #{OUT}" }
+FileUtils.rm_rf TMP
+FileUtils.mkdir_p TMP
+FileUtils.touch(OUT)
 
-at_exit do
-  Process.kill(:TERM, FWD)
-  Process.kill(:TERM, NCC)
-end
+FWD = fork { exec "#{root}/bin/fwd-rb --flush 10000:2 -F tcp://0.0.0.0:7291 --path #{TMP} -v" }
+NCC = fork { exec "nc -vlp 7291 > #{OUT}" }
 
-sleep(3)
+sleep(5)
 
 EVENTS = 10_000_000
 LENGTH = 100
 DATA   = "A" * LENGTH
-
-ds = Benchmark.realtime do
-  sock = TCPSocket.new "127.0.0.1", 7289
-  EVENTS.times { sock.write DATA }
-  sock.close
+CCUR   = 5
+
+ds   = Benchmark.realtime do
+  (1..CCUR).map do
+    fork do
+      sock = TCPSocket.new "127.0.0.1", 7289
+      (EVENTS / CCUR).times { sock.write DATA }
+      sock.close
+    end
+  end.each {|t| Process.wait(t) }
 end
 
 rs = Benchmark.realtime do
@@ -44,3 +46,6 @@ puts "--> Completed in  : #{(ds + rs).round(1)}s"
 puts "--> FWD RSS       : #{(`ps -o rss= -p #{FWD}`.to_f / 1024).round(1)}M"
 puts "--> Processed     : #{EVENTS} events"
 puts "--> Written       : #{(OUT.size / 1024.0 / 1024.0).round(1)}M"
+
+Process.kill(:TERM, FWD)
+Process.kill(:TERM, NCC)

data/fwd.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |s|
   s.name        = File.basename(__FILE__, '.gemspec')
   s.summary     = "fwd >>"
   s.description = "The minimalistic stream forwarder"
-  s.version     = "0.3.2"
+  s.version     = "0.3.3"
 
   s.authors     = ["Black Square Media"]
   s.email       = "info@blacksquaremedia.com"

data/lib/fwd.rb

@@ -87,7 +87,8 @@ class Fwd
   # Starts the server
   def listen!
     logger.info "Starting server on #{@bind}"
-    EM.start_server @bind.host, @bind.port, Fwd::Input, self
+    buffer = Fwd::Buffer.new(self)
+    EM.start_server @bind.host, @bind.port, Fwd::Input, self, buffer
   end
 
   # Initiates flush

data/lib/fwd/buffer.rb

@@ -13,9 +13,7 @@ class Fwd::Buffer
     @rate     = (core.opts[:flush_rate] || 10_000).to_i
     @limit    = [core.opts[:buffer_limit].to_i, MAX_LIMIT].reject(&:zero?).min
     @count    = 0
-
     reschedule!
-    rotate!
   end
 
   # @param [String] data binary data
@@ -50,7 +48,8 @@ class Fwd::Buffer
     end
 
     @fd = new_file
-  rescue Errno::ENOENT
+  rescue Errno::ENOENT => e
+    logger.warn "Rotation delayed: #{e.message}"
   end
 
   # @return [Boolean] true if rotation is due

data/lib/fwd/input.rb

@@ -5,13 +5,10 @@ class Fwd::Input < EM::Connection
   attr_reader :core, :buffer
 
   # @param [Fwd] core
-  # @param [Hash] opts additional opts
-  def initialize(core)
-    @core = core
-  end
-
-  def post_init
-    @buffer = Fwd::Buffer.new(core)
+  # @param [Fwd::Buffer] buffer
+  def initialize(core, buffer)
+    @core   = core
+    @buffer = buffer
   end
 
   # When receiving data, concat it to the buffer

data/lib/fwd/output.rb

@@ -2,7 +2,8 @@ class Fwd::Output
   extend Forwardable
   def_delegators :core, :logger, :root, :prefix
 
-  RESCUABLE = [
+  CHUNK_SIZE = 16 * 1024
+  RESCUABLE  = [
     Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::EHOSTUNREACH, Errno::EPIPE,
     Errno::ENETUNREACH, Errno::ENETDOWN, Errno::EINVAL, Errno::ETIMEDOUT,
     IOError, EOFError
@@ -22,19 +23,26 @@ class Fwd::Output
 
   # Callback
   def forward!
-    Dir[root.join("#{prefix}.*.closed")].each do |file|
-      ok = reserve(file) do |data|
-        logger.debug { "Flushing #{File.basename(file)}, #{data.size.fdiv(1024).round} kB" }
-        write(data)
+    return if @forwarding
+
+    @forwarding = true
+    begin
+      Dir[root.join("#{prefix}.*.closed")].sort.each do |file|
+        ok = reserve(file) do |io|
+          logger.debug { "Flushing #{File.basename(io.path)}, #{io.size.fdiv(1024).round} kB" }
+          write(io)
+        end
+        ok or break
       end
-      break unless ok
+    ensure
+      @forwarding = false
     end
   end
 
-  # @param [String] binary data
-  def write(data)
+  # @param [IO] io source stream
+  def write(io)
     pool.any? do |backend|
-      forward(backend, data)
+      forward(backend, io)
     end
   end
 
@@ -46,7 +54,11 @@ class Fwd::Output
       target = Pathname.new(file.sub(/\.closed$/, ".reserved"))
       FileUtils.mv file, target.to_s
 
-      result = yield(target.read)
+      result = false
+      target.open("r") do |io|
+        result = yield(io)
+      end
+
       if result
         target.unlink
       else
@@ -55,12 +67,17 @@ class Fwd::Output
       end
 
       result
-    rescue Errno::ENOENT
+    rescue Errno::ENOENT => e
       # Ignore if file was alread flushed by another process
+      logger.warn "Flushing of #{File.basename(file)} postponed: #{e.message}"
     end
 
-    def forward(backend, data)
-      backend.write(data) && true
+    def forward(backend, io)
+      io.rewind
+      until io.eof?
+        backend.write(io.read(CHUNK_SIZE))
+      end
+      true
     rescue *RESCUABLE => e
       logger.error "Backend #{backend} failed: #{e.class.name} #{e.message}"
       backend.close

data/spec/fwd/buffer_spec.rb

@@ -7,14 +7,9 @@ describe Fwd::Buffer do
   end
 
   let(:buffer) { described_class.new core }
-  let(:timer)  { mock("Timer", cancel: true) }
   subject      { buffer }
-  before do
-    EM.stub add_periodic_timer: timer
-  end
 
   its(:root)     { should == root }
-  its(:root)     { should be_exist }
   its(:prefix)   { should == "buffer" }
   its(:core)     { should be(core) }
   its(:count)    { should be(0) }
@@ -22,21 +17,26 @@ describe Fwd::Buffer do
   its(:rate)     { should be(20) }
   its(:limit)    { should be(2048) }
   its(:timer)    { should be(timer) }
-  its(:fd)       { should be_instance_of(File) }
+  its(:fd)       { should be_nil }
   its(:logger)   { should be(Fwd.logger) }
 
   describe "concat" do
     it 'should concat data' do
-      lambda {
-        subject.concat("x" * 1024)
-      }.should change {
-        subject.fd.size
-      }.by(1024)
+      subject.concat("x" * 1024)
+      subject.fd.size.should == 1024
+    end
+
+    it 'should create the path' do
+      lambda { buffer.concat("x") }.should change { buffer.root.exist? }.to(true)
+    end
+
+    it 'should rotate' do
+      lambda { buffer.concat("x") }.should change { buffer.fd }.to(instance_of(File))
     end
   end
 
   describe "rotate" do
-    before  { buffer }
+    before  { buffer.send(:rotate!) }
     subject { lambda { buffer.rotate! } }
 
     it 'should trigger when buffer limit is reached' do
@@ -58,7 +58,7 @@ describe Fwd::Buffer do
     end
 
     describe "when unchanged" do
-      it { should_not change { buffer.fd.path } }
+      it { should_not change { buffer.fd.size }.from(0) }
       it { should_not change { files.size } }
     end
   end

data/spec/fwd/input_spec.rb

@@ -2,20 +2,18 @@ require 'spec_helper'
 
 describe Fwd::Input do
 
+  let(:buffer) { Fwd::Buffer.new(core) }
   subject do
     input = described_class.allocate
-    input.send(:initialize, core)
+    input.send(:initialize, core, buffer)
     input
   end
+
   before { EM.stub :add_periodic_timer }
 
   it { should be_a(EM::Connection) }
-  its(:buffer) { should be_nil }
+  its(:core) { should be(core) }
+  its(:buffer) { should be(buffer) }
   its(:logger) { should be(Fwd.logger) }
 
-  describe "post init" do
-    before { subject.post_init }
-    its(:buffer) { should be_instance_of(Fwd::Buffer) }
-  end
-
 end

data/spec/fwd/output_spec.rb

@@ -48,30 +48,34 @@ describe Fwd::Output do
   end
 
   describe "writing" do
+    def write(data)
+      subject.write(StringIO.new(data))
+    end
 
     it 'should forward data to backends' do
       servers(7291, 7292) do
-        subject.write("A").should be(true)
-        subject.write("B").should be(true)
-        subject.write("C").should be(true)
-        subject.write("D").should be(true)
+        write("A").should be(true)
+        write("B").should be(true)
+        write("C").should be(true)
+        write("D").should be(true)
       end.should == { 7291=>"BD", 7292=>"AC" }
     end
 
     it 'should handle partial fallouts' do
       servers(7291) do
-        subject.write("A").should be(true)
-        subject.write("B").should be(true)
-        subject.write("C").should be(true)
-        subject.write("D").should be(true)
+        write("A").should be(true)
+        write("B").should be(true)
+        write("C").should be(true)
+        write("D").should be(true)
+        sleep(1)
       end.should == { 7291=>"ABCD" }
     end
 
     it 'should handle full fallouts' do
-      subject.write("A").should be(false)
-      subject.write("B").should be(false)
-      subject.write("C").should be(false)
-      subject.write("D").should be(false)
+      write("A").should be(false)
+      write("B").should be(false)
+      write("C").should be(false)
+      write("D").should be(false)
     end
 
   end

data/spec/spec_helper.rb

@@ -26,6 +26,10 @@ module Fwd::TestHelper
       forward: ["tcp://127.0.0.1:7291", "tcp://127.0.0.1:7292"]
   end
 
+  def timer
+    @_timer ||= mock("Timer", cancel: true)
+  end
+
 end
 
 RSpec.configure do |c|
@@ -35,5 +39,6 @@ RSpec.configure do |c|
   end
   c.before(:each) do
     FileUtils.rm_rf root.to_s
+    EM.stub add_periodic_timer: timer
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fwd
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-20 00:00:00.000000000 Z
+date: 2013-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine-le