fwd 0.3.1 → 0.3.2

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fwd (0.3.1)
+    fwd (0.3.2)
       connection_pool
       eventmachine-le
       servolux

data/benchmark/performance.rb

@@ -1,40 +1,46 @@
 #!/usr/bin/env ruby
 
-$:.unshift(File.expand_path('../../lib', __FILE__))
-
-require 'bundler/setup'
+require 'pathname'
 require 'benchmark'
-require 'tempfile'
-require 'fwd'
+require 'socket'
+require 'fileutils'
 
 root = Pathname.new(File.expand_path('../..', __FILE__))
-FileUtils.rm_rf root.join("tmp/benchmark")
-FileUtils.mkdir_p root.join("tmp/benchmark")
+tmp  = root.join("tmp/benchmark")
+FileUtils.rm_rf tmp
+FileUtils.mkdir_p tmp
 
-EVENTS = 10_000_000
-DATA   = "A" * 64
-OUTF   = root.join('tmp/benchmark/out.txt')
+OUT = tmp.join('out.txt')
+FWD = fork { exec "#{root}/bin/fwd-rb --flush 10000:2 -F tcp://0.0.0.0:7291 --path #{tmp} -v" }
+NCC = fork { exec "nc -vlp 7291 > #{OUT}" }
 
-COLL   = fork do
-  `nc -vlp 7291 > #{OUTF}`
-  sleep
+at_exit do
+  Process.kill(:TERM, FWD)
+  Process.kill(:TERM, NCC)
 end
-EMIT   = fork do
+
+sleep(3)
+
+EVENTS = 10_000_000
+LENGTH = 100
+DATA   = "A" * LENGTH
+
+ds = Benchmark.realtime do
   sock = TCPSocket.new "127.0.0.1", 7289
   EVENTS.times { sock.write DATA }
   sock.close
 end
 
-at_exit do
-  Process.kill(:TERM, COLL)
-  Process.kill(:TERM, EMIT)
+rs = Benchmark.realtime do
+  while OUT.size < EVENTS * LENGTH
+    sleep(1)
+    puts "--> Written       : #{(OUT.size / 1024.0 / 1024.0).round(1)}M of #{(EVENTS * LENGTH / 1024.0 / 1024.0).round(1)}M"
+  end
 end
 
-until OUTF.exist?
-  sleep(1)
-end
-
-while OUTF.size < EVENTS * DATA.size
-  sleep(1)
-  puts "Written #{(OUTF.size / 1024.0 / 1024.0).round(1)}M"
-end
+sleep(3)
+puts "--> Dispatched in : #{ds.round(1)}s"
+puts "--> Completed in  : #{(ds + rs).round(1)}s"
+puts "--> FWD RSS       : #{(`ps -o rss= -p #{FWD}`.to_f / 1024).round(1)}M"
+puts "--> Processed     : #{EVENTS} events"
+puts "--> Written       : #{(OUT.size / 1024.0 / 1024.0).round(1)}M"

data/fwd.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |s|
   s.name        = File.basename(__FILE__, '.gemspec')
   s.summary     = "fwd >>"
   s.description = "The minimalistic stream forwarder"
-  s.version     = "0.3.1"
+  s.version     = "0.3.2"
 
   s.authors     = ["Black Square Media"]
   s.email       = "info@blacksquaremedia.com"

data/lib/fwd/buffer.rb

@@ -2,7 +2,7 @@ class Fwd::Buffer
   extend Forwardable
   def_delegators :core, :root, :prefix, :logger
 
-  MAX_SIZE = 64 * 1024 * 1024 # 64M
+  MAX_LIMIT = 64 * 1024 * 1024 # 64M
   attr_reader :core, :interval, :rate, :count, :limit, :timer, :fd
 
   # Constructor
@@ -11,7 +11,7 @@ class Fwd::Buffer
     @core     = core
     @interval = (core.opts[:flush_interval] || 60).to_i
     @rate     = (core.opts[:flush_rate] || 10_000).to_i
-    @limit    = (core.opts[:flush_limit] || 0).to_i
+    @limit    = [core.opts[:buffer_limit].to_i, MAX_LIMIT].reject(&:zero?).min
     @count    = 0
 
     reschedule!
@@ -37,8 +37,7 @@ class Fwd::Buffer
 
   # @return [Boolean] true if flush is due
   def flush?
-    return unless @fd
-    (@rate > 0 && @count >= @rate) || (@limit > 0 && @fd.size >= @limit)
+    @rate > 0 && @count >= @rate
   end
 
   # (Force) rotate buffer file
@@ -46,7 +45,7 @@ class Fwd::Buffer
     return if @fd && @fd.size.zero?
 
     if @fd
-      logger.debug { "Rotate #{File.basename(@fd.path)} (#{@fd.size / 1024} kB)" }
+      logger.debug { "Rotating #{File.basename(@fd.path)}, #{@fd.size / 1024} kB" }
       FileUtils.mv(@fd.path, @fd.path.sub(/\.open$/, ".closed"))
     end
 
@@ -56,7 +55,7 @@ class Fwd::Buffer
 
   # @return [Boolean] true if rotation is due
   def rotate?
-    @fd.nil? || @fd.size > MAX_SIZE
+    @fd.nil? || @fd.size >= @limit
   rescue Errno::ENOENT
     false
   end

data/lib/fwd/cli.rb

@@ -32,13 +32,12 @@ class Fwd::CLI < Hash
         update forward: uris.map {|uri| URI.parse(uri).to_s }
       end
 
-      o.on("-f", "--flush L:M:N",
+      o.on("-f", "--flush M:N",
         "Flush after an interval of N seconds, " <<
         "or after receiving M messages, " <<
-        "or after the limit of L bytes was reached. " <<
-        "Default: 0:10000:60") do |values|
-        l,m,n = values.split(":").map(&:to_i)
-        update flush_limit: l.to_i, flush_rate: m.to_i, flush_interval: n.to_i
+        "Default: 10000:60") do |values|
+        m,n = values.split(":").map(&:to_i)
+        update flush_rate: m.to_i, flush_interval: n.to_i
       end
 
       o.on("--path PATH", "Root path for storage. Default: ./tmp") do |path|
@@ -49,6 +48,10 @@ class Fwd::CLI < Hash
         update prefix: prefix
       end
 
+      o.on("-v", "--verbose", "Enable verbose logging.") do |_|
+        Fwd.logger.level = Logger::DEBUG
+      end
+
       o.separator ""
       o.on_tail("-h", "--help", "Show this message") do
         puts o

data/lib/fwd.rb

@@ -17,7 +17,9 @@ class Fwd
 
     # [Logger] logger instance
     def logger
-      @logger ||= ::Logger.new(STDOUT)
+      @logger ||= ::Logger.new(STDOUT).tap do |l|
+        l.level = ::Logger::INFO
+      end
     end
 
   end
@@ -43,8 +45,8 @@ class Fwd
   # @option opts [String]  prefix buffer file prefix
   # @option opts [URI] bind the endpoint to listen to
   # @option opts [Array<URI>] forward the endpoints to forward to
-  # @option opts [Integer] flush_limit flush after L messages
-  # @option opts [Integer] flush_rate flush after M messages
+  # @option opts [Integer] buffer_limit limit buffer files to N bytes
+  # @option opts [Integer] flush_rate flush after N messages
   # @option opts [Integer] flush_interval flush after N seconds
   def initialize(opts = {})
     @bind   = URI.parse(opts[:bind] || "tcp://0.0.0.0:7289")

data/spec/fwd/buffer_spec.rb

@@ -39,6 +39,11 @@ describe Fwd::Buffer do
     before  { buffer }
     subject { lambda { buffer.rotate! } }
 
+    it 'should trigger when buffer limit is reached' do
+      lambda { buffer.concat("x" * 2048) }.should_not change { buffer.fd.path }
+      lambda { buffer.concat("x") }.should change { buffer.fd.path }
+    end
+
     describe "when changed" do
       before { buffer.concat("x" * 1024) }
 
@@ -67,11 +72,6 @@ describe Fwd::Buffer do
       lambda { subject.concat("x") }.should change { subject.count }.from(19).to(0)
     end
 
-    it 'should trigger when flush limit is reached' do
-      subject.concat("x" * 1024)
-      lambda { subject.concat("x" * 1024) }.should change { subject.count }.from(1).to(0)
-    end
-
     it 'should reset count' do
       3.times { subject.concat("x") }
       lambda { subject.flush! }.should change { subject.count }.from(3).to(0)

data/spec/fwd/cli_spec.rb

@@ -8,7 +8,7 @@ describe Fwd::CLI do
       "--prefix", "prefix",
       "--bind", "tcp://127.0.0.1:7289",
       "--forward", "tcp://1.2.3.4:1234,tcp://1.2.3.5:1235",
-      "--flush", "30:1200:90",
+      "--flush", "1200:90",
     ]
   end
 
@@ -17,7 +17,6 @@ describe Fwd::CLI do
   its([:prefix]) { should == "prefix" }
   its([:bind]) { should == "tcp://127.0.0.1:7289" }
   its([:forward]) { should == ["tcp://1.2.3.4:1234", "tcp://1.2.3.5:1235"] }
-  its([:flush_limit]) { should == 30 }
   its([:flush_rate]) { should == 1200 }
   its([:flush_interval]) { should == 90 }
   its(:core) { should be_instance_of(Fwd) }

data/spec/spec_helper.rb

@@ -22,7 +22,7 @@ module Fwd::TestHelper
     @_core ||= Fwd.new \
       path: root,
       flush_rate: 20,
-      flush_limit: 2048,
+      buffer_limit: 2048,
       forward: ["tcp://127.0.0.1:7291", "tcp://127.0.0.1:7292"]
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fwd
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
   prerelease: 
 platform: ruby
 authors: