fwd 0.3.3 → 0.4.0

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 1.9.3

data/Gemfile

@@ -1,2 +1,2 @@
-source :rubygems
+source "https://rubygems.org"
 gemspec

data/Gemfile.lock

@@ -7,22 +7,22 @@ PATH
       servolux
 
 GEM
-  remote: http://rubygems.org/
+  remote: https://rubygems.org/
   specs:
     connection_pool (1.0.0)
-    diff-lcs (1.1.3)
+    diff-lcs (1.2.1)
     eventmachine-le (1.1.4)
     rake (10.0.3)
-    rspec (2.12.0)
-      rspec-core (~> 2.12.0)
-      rspec-expectations (~> 2.12.0)
-      rspec-mocks (~> 2.12.0)
-    rspec-core (2.12.2)
-    rspec-expectations (2.12.1)
-      diff-lcs (~> 1.1.3)
-    rspec-mocks (2.12.2)
+    rspec (2.13.0)
+      rspec-core (~> 2.13.0)
+      rspec-expectations (~> 2.13.0)
+      rspec-mocks (~> 2.13.0)
+    rspec-core (2.13.0)
+    rspec-expectations (2.13.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.13.0)
     servolux (0.10.0)
-    yard (0.8.4.1)
+    yard (0.8.5.2)
 
 PLATFORMS
   ruby

data/benchmark/performance.rb

@@ -13,16 +13,14 @@ FileUtils.rm_rf TMP
 FileUtils.mkdir_p TMP
 FileUtils.touch(OUT)
 
-FWD = fork { exec "#{root}/bin/fwd-rb --flush 10000:2 -F tcp://0.0.0.0:7291 --path #{TMP} -v" }
-NCC = fork { exec "nc -vlp 7291 > #{OUT}" }
-
-sleep(5)
-
+FWD    = spawn "#{root}/bin/fwd-rb --flush 10000:2 -F tcp://0.0.0.0:7291 --path #{TMP} -v"
+NCC    = spawn "nc -kl 7291", out: [OUT, "w"]
 EVENTS = 10_000_000
-LENGTH = 100
-DATA   = "A" * LENGTH
+DATA   = (("A".."Z").to_a + ("a".."z").to_a).join + "\n"
+LENGTH = DATA.size
 CCUR   = 5
 
+sleep(5)
 ds   = Benchmark.realtime do
   (1..CCUR).map do
     fork do

data/fwd.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |s|
   s.name        = File.basename(__FILE__, '.gemspec')
   s.summary     = "fwd >>"
   s.description = "The minimalistic stream forwarder"
-  s.version     = "0.3.3"
+  s.version     = "0.4.0"
 
   s.authors     = ["Black Square Media"]
   s.email       = "info@blacksquaremedia.com"

data/lib/fwd.rb

@@ -11,19 +11,6 @@ require 'servolux'
 class Fwd
   FLUSH = "\000>>"
 
-  class << self
-
-    attr_writer :logger
-
-    # [Logger] logger instance
-    def logger
-      @logger ||= ::Logger.new(STDOUT).tap do |l|
-        l.level = ::Logger::INFO
-      end
-    end
-
-  end
-
   # @attr_reader [URI] uri to bind to
   attr_reader :bind
 
@@ -36,6 +23,9 @@ class Fwd
   # @attr_reader [Fwd::Output] output
   attr_reader :output
 
+  # @attr_reader [Logger] logger
+  attr_reader :logger
+
   # @attr_reader [Hash] opts
   attr_reader :opts
 
@@ -49,18 +39,19 @@ class Fwd
   # @option opts [Integer] flush_rate flush after N messages
   # @option opts [Integer] flush_interval flush after N seconds
   def initialize(opts = {})
+    @opts   = opts
     @bind   = URI.parse(opts[:bind] || "tcp://0.0.0.0:7289")
     @root   = Pathname.new(opts[:path] || "tmp")
     @prefix = opts[:prefix] || "buffer"
-    @opts   = opts
+    @logger = ::Logger.new(opts[:log] || STDOUT)
+    @logger.level = opts[:log_level] || ::Logger::INFO
     @output = Fwd::Output.new(self)
   end
 
   # Starts the loop
   def run!
-    $0 = "fwd-rb (output)"
-
     @piper = ::Servolux::Piper.new('rw')
+
     at_exit do
       @piper.signal("TERM")
     end
@@ -71,16 +62,8 @@ class Fwd
     end
 
     @piper.parent do
-      loop do
-        sleep(0.1)
-        case val = @piper.gets()
-        when FLUSH
-          output.forward!
-        else
-          logger.error "Received unknown message #{val.class.name} "
-          exit
-        end
-      end
+      $0 = "fwd-rb (output)"
+      output_loop!
     end
   end
 
@@ -91,6 +74,20 @@ class Fwd
     EM.start_server @bind.host, @bind.port, Fwd::Input, self, buffer
   end
 
+  # Starts the output loop
+  def output_loop!
+    loop do
+      sleep(0.1)
+      case val = @piper.gets()
+      when FLUSH
+        @output.forward!
+      else
+        logger.error "Received unknown message #{val.class.name}: #{val.inspect}"
+        exit
+      end
+    end
+  end
+
   # Initiates flush
   def flush!
     @piper.child do
@@ -98,11 +95,6 @@ class Fwd
     end
   end
 
-  # [Logger] logger instance
-  def logger
-    self.class.logger
-  end
-
 end
 
 %w|buffer output backend input pool cli|.each do |name|

data/lib/fwd/backend.rb

@@ -5,8 +5,8 @@ class Fwd::Backend
     @url = URI(url)
   end
 
-  def write(data)
-    sock.write(data)
+  def stream(file)
+    IO.copy_stream file.to_s, sock
   end
 
   def close

data/lib/fwd/buffer.rb

@@ -13,12 +13,15 @@ class Fwd::Buffer
     @rate     = (core.opts[:flush_rate] || 10_000).to_i
     @limit    = [core.opts[:buffer_limit].to_i, MAX_LIMIT].reject(&:zero?).min
     @count    = 0
+    cleanup!
     reschedule!
+
+    at_exit { rotate! }
   end
 
   # @param [String] data binary data
   def concat(data)
-    rotate! if rotate?
+    rotate! if limit_reached?
     @fd.write(data)
     @count += 1
     flush! if flush?
@@ -43,8 +46,8 @@ class Fwd::Buffer
     return if @fd && @fd.size.zero?
 
     if @fd
-      logger.debug { "Rotating #{File.basename(@fd.path)}, #{@fd.size / 1024} kB" }
-      FileUtils.mv(@fd.path, @fd.path.sub(/\.open$/, ".closed"))
+      close(@fd.path)
+      logger.debug { "Rotated #{File.basename(@fd.path)}, #{@fd.size / 1024}k" }
     end
 
     @fd = new_file
@@ -52,8 +55,8 @@ class Fwd::Buffer
     logger.warn "Rotation delayed: #{e.message}"
   end
 
-  # @return [Boolean] true if rotation is due
-  def rotate?
+  # @return [Boolean] true if limit reached
+  def limit_reached?
     @fd.nil? || @fd.size >= @limit
   rescue Errno::ENOENT
     false
@@ -72,6 +75,16 @@ class Fwd::Buffer
       file
     end
 
+    def close(file)
+      FileUtils.mv(file, file.sub(/\.open$/, ".closed"))
+    end
+
+    def cleanup!
+      Dir[root.join("#{prefix}.*.open")].each do |file|
+        File.size(file).zero? ? File.unlink(file) : close(file)
+      end
+    end
+
     def reschedule!
       return unless @interval > 0
 
@@ -80,7 +93,7 @@ class Fwd::Buffer
     end
 
     def generate_name
-      [prefix, Time.now.utc.strftime("%Y%m%d%H%m%s"), SecureRandom.hex(4)].join(".")
+      [prefix, (Time.now.utc.to_f * 1000).round, SecureRandom.hex(2)].join(".")
     end
 
 end

data/lib/fwd/cli.rb

@@ -48,8 +48,12 @@ class Fwd::CLI < Hash
         update prefix: prefix
       end
 
-      o.on("-v", "--verbose", "Enable verbose logging.") do |_|
-        Fwd.logger.level = Logger::DEBUG
+      o.on("-l", "--log PATH", "Custom log path. Default: STDOUT") do |path|
+        update log: path
+      end
+
+      o.on("-v", "--verbose", "Enable verbose logging.") do
+        update log_level: ::Logger::DEBUG
       end
 
       o.separator ""

data/lib/fwd/output.rb

@@ -2,7 +2,6 @@ class Fwd::Output
   extend Forwardable
   def_delegators :core, :logger, :root, :prefix
 
-  CHUNK_SIZE = 16 * 1024
   RESCUABLE  = [
     Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::EHOSTUNREACH, Errno::EPIPE,
     Errno::ENETUNREACH, Errno::ENETDOWN, Errno::EINVAL, Errno::ETIMEDOUT,
@@ -26,23 +25,24 @@ class Fwd::Output
     return if @forwarding
 
     @forwarding = true
-    begin
-      Dir[root.join("#{prefix}.*.closed")].sort.each do |file|
-        ok = reserve(file) do |io|
-          logger.debug { "Flushing #{File.basename(io.path)}, #{io.size.fdiv(1024).round} kB" }
-          write(io)
-        end
-        ok or break
+    while (q = closed_files) && (file = q.shift)
+      ok = reserve(file) do |reserved|
+        start   = Time.now
+        success = stream_file(reserved)
+        real    = Time.now - start
+        logger.info { "Flushed #{reserved.basename}, #{reserved.size.fdiv(1024).round}k in #{real.round(1)}s (Q: #{q.size})" }
+        success
       end
-    ensure
-      @forwarding = false
+      ok || break
     end
+  ensure
+    @forwarding = false
   end
 
-  # @param [IO] io source stream
-  def write(io)
+  # @param [Pathname] file file to stream
+  def stream_file(file)
     pool.any? do |backend|
-      forward(backend, io)
+      stream_to(backend, file)
     end
   end
 
@@ -54,29 +54,22 @@ class Fwd::Output
       target = Pathname.new(file.sub(/\.closed$/, ".reserved"))
       FileUtils.mv file, target.to_s
 
-      result = false
-      target.open("r") do |io|
-        result = yield(io)
-      end
-
-      if result
+      success = yield(target)
+      if success
         target.unlink
       else
-        logger.error "Flushing of #{target} failed."
-        FileUtils.mv target.to_s, target.to_s.sub(/\.reserved$/, ".closed")
+        logger.error "Flushing #{File.basename(file)} failed"
+        FileUtils.mv target.to_s, file
       end
 
-      result
+      success
     rescue Errno::ENOENT => e
       # Ignore if file was alread flushed by another process
-      logger.warn "Flushing of #{File.basename(file)} postponed: #{e.message}"
+      logger.warn "Flushing #{File.basename(file)} postponed: #{e.message}"
     end
 
-    def forward(backend, io)
-      io.rewind
-      until io.eof?
-        backend.write(io.read(CHUNK_SIZE))
-      end
+    def stream_to(backend, file)
+      backend.stream(file)
       true
     rescue *RESCUABLE => e
       logger.error "Backend #{backend} failed: #{e.class.name} #{e.message}"
@@ -84,4 +77,8 @@ class Fwd::Output
       false
     end
 
+    def closed_files
+      Dir[root.join("#{prefix}.*.closed")].sort
+    end
+
 end

data/spec/fwd/buffer_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 describe Fwd::Buffer do
 
   def files(glob = "*")
-    Dir[root.join(glob)]
+    Dir[root.join(glob)].map {|f| File.basename(f) }.sort
   end
 
   let(:buffer) { described_class.new core }
@@ -18,7 +18,22 @@ describe Fwd::Buffer do
   its(:limit)    { should be(2048) }
   its(:timer)    { should be(timer) }
   its(:fd)       { should be_nil }
-  its(:logger)   { should be(Fwd.logger) }
+  its(:logger)   { should be(core.logger) }
+
+  it 'should clean up existing files' do
+    FileUtils.mkdir_p(root.to_s)
+    f1, f2 = "buffer.0.blank.open", "buffer.0.filled.open"
+    root.join(f1).open("wb") {}
+    root.join(f2).open("wb") {|f| f << "A" }
+    lambda { subject }.should change { files }.from([f1, f2]).to(["buffer.0.filled.closed"])
+  end
+
+  it 'should generate unique buffer file names' do
+    Time.stub now: Time.at(1313131313.2345678)
+    SecureRandom.stub hex: "6a7b8c9d"
+    buffer.concat("x")
+    buffer.fd.path.should == root.join("buffer.1313131313235.6a7b8c9d.open").to_s
+  end
 
   describe "concat" do
     it 'should concat data' do
@@ -51,7 +66,7 @@ describe Fwd::Buffer do
       it { should change { files.size }.by(1) }
 
       it 'should archive previous file' do
-        previous = buffer.fd.path
+        previous = File.basename(buffer.fd.path)
         subject.call
         files.should include(previous.sub("open", "closed").to_s)
       end

data/spec/fwd/cli_spec.rb

@@ -9,6 +9,8 @@ describe Fwd::CLI do
       "--bind", "tcp://127.0.0.1:7289",
       "--forward", "tcp://1.2.3.4:1234,tcp://1.2.3.5:1235",
       "--flush", "1200:90",
+      "--log", "/dev/null",
+      "--verbose",
     ]
   end
 
@@ -19,6 +21,8 @@ describe Fwd::CLI do
   its([:forward]) { should == ["tcp://1.2.3.4:1234", "tcp://1.2.3.5:1235"] }
   its([:flush_rate]) { should == 1200 }
   its([:flush_interval]) { should == 90 }
+  its([:log]) { should == "/dev/null" }
+  its([:log_level]) { should == 0 }
   its(:core) { should be_instance_of(Fwd) }
 
 end

data/spec/fwd/input_spec.rb

@@ -14,6 +14,6 @@ describe Fwd::Input do
   it { should be_a(EM::Connection) }
   its(:core) { should be(core) }
   its(:buffer) { should be(buffer) }
-  its(:logger) { should be(Fwd.logger) }
+  its(:logger) { should be(core.logger) }
 
 end

data/spec/fwd/output_spec.rb

@@ -2,6 +2,7 @@ require 'spec_helper'
 
 describe Fwd::Output do
 
+  before       { FileUtils.mkdir_p root.to_s }
   let(:output) { described_class.new core }
   subject { output }
 
@@ -47,35 +48,37 @@ describe Fwd::Output do
     subject.pool.checkout {|c| c.should be_instance_of(Fwd::Backend) }
   end
 
-  describe "writing" do
-    def write(data)
-      subject.write(StringIO.new(data))
+  describe "streaming data" do
+
+    def stream(data)
+      tmp = root.join("temp.file")
+      tmp.open("wb") {|f| f << data }
+      subject.stream_file(tmp)
     end
 
     it 'should forward data to backends' do
       servers(7291, 7292) do
-        write("A").should be(true)
-        write("B").should be(true)
-        write("C").should be(true)
-        write("D").should be(true)
+        stream("A").should be(true)
+        stream("B").should be(true)
+        stream("C").should be(true)
+        stream("D").should be(true)
       end.should == { 7291=>"BD", 7292=>"AC" }
     end
 
     it 'should handle partial fallouts' do
       servers(7291) do
-        write("A").should be(true)
-        write("B").should be(true)
-        write("C").should be(true)
-        write("D").should be(true)
-        sleep(1)
+        stream("A").should be(true)
+        stream("B").should be(true)
+        stream("C").should be(true)
+        stream("D").should be(true)
       end.should == { 7291=>"ABCD" }
     end
 
     it 'should handle full fallouts' do
-      write("A").should be(false)
-      write("B").should be(false)
-      write("C").should be(false)
-      write("D").should be(false)
+      stream("A").should be(false)
+      stream("B").should be(false)
+      stream("C").should be(false)
+      stream("D").should be(false)
     end
 
   end
@@ -83,7 +86,7 @@ describe Fwd::Output do
   describe "forwarding" do
 
     def write(file)
-      file.open("w") {|f| f << "x" }
+      file.open("wb") {|f| f << "x" }
       file
     end
 
@@ -91,14 +94,18 @@ describe Fwd::Output do
       Dir[root.join(glob)].map {|f| File.basename f }.sort
     end
 
-    before    { subject.stub! write: true }
-    before    { FileUtils.mkdir_p root.to_s }
+    before    { subject.stub! stream_file: true }
     let!(:f1) { write root.join("buffer.1.closed") }
     let!(:f2) { write root.join("buffer.2.open") }
     let!(:f3) { write root.join("buffer.3.closed") }
 
-    it 'should write the data' do
-      subject.should_receive(:write).twice
+    it 'should send the data' do
+      subject.should_receive(:stream_file).twice.and_return(true)
+      subject.forward!
+    end
+
+    it 'should stop on first failure' do
+      subject.should_receive(:stream_file).once.and_return(false)
       subject.forward!
     end
 
@@ -108,9 +115,9 @@ describe Fwd::Output do
       }.to(["buffer.2.open"])
     end
 
-    it 'should handle failures files' do
-      subject.should_receive(:write).and_return(true)
-      subject.should_receive(:write).and_return(false)
+    it 'should handle revert failed files' do
+      subject.should_receive(:stream_file).and_return(true)
+      subject.should_receive(:stream_file).and_return(false)
 
       lambda { subject.forward! }.should change {
         files

data/spec/fwd_spec.rb

@@ -5,6 +5,7 @@ describe Fwd do
   subject      { core }
 
   its(:logger) { should be_instance_of(::Logger) }
+  its("logger.level") { should == 0 }
   its(:root)   { should be_instance_of(::Pathname) }
   its(:root)   { should == root }
   its(:bind)   { should be_instance_of(URI::Generic) }

data/spec/spec_helper.rb

@@ -21,6 +21,8 @@ module Fwd::TestHelper
   def core
     @_core ||= Fwd.new \
       path: root,
+      log:  "/dev/null",
+      log_level: Logger::DEBUG,
       flush_rate: 20,
       buffer_limit: 2048,
       forward: ["tcp://127.0.0.1:7291", "tcp://127.0.0.1:7292"]
@@ -34,9 +36,6 @@ end
 
 RSpec.configure do |c|
   c.include(Fwd::TestHelper)
-  c.before(:suite) do
-    Fwd.logger = Logger.new("/dev/null")
-  end
   c.before(:each) do
     FileUtils.rm_rf root.to_s
     EM.stub add_periodic_timer: timer

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fwd
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-21 00:00:00.000000000 Z
+date: 2013-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine-le
@@ -131,6 +131,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - Gemfile
 - Gemfile.lock
 - Rakefile