io-tail 0.0.1

data/lib/io/tail/group.rb

@@ -0,0 +1,124 @@
+require 'thread'
+
+class IO
+  module Tail
+    # This class can be used to coordinate tailing of many files, which have
+    # been added to the group.
+    class Group
+      # Creates a new IO::Tail::Group instance.
+      #
+      # The following options can be given as arguments:
+      # :files:: an array of files (or filenames to open) that are placed into
+      #          the group.
+      def initialize(opts = {})
+        @tailers = ThreadGroup.new
+        if files = opts[:files]
+          Array(files).each { |file| add file }
+        end
+      end
+
+      # Creates a group for +files+ (IO instances or filename strings).
+      def self.[](*files)
+        new(:files => files)
+      end
+
+      # Add a file (IO instance) or filename (responding to to_str) to this
+      # group.
+      def add(file_or_filename)
+        if file_or_filename.is_a?(IO::Tail::Tailable)
+          add_tailable file_or_filename
+        elsif file_or_filename.respond_to?(:to_str)
+          add_filename file_or_filename
+        end
+      end
+
+      alias << add
+
+      # Add the IO instance +file+ to this group.
+      def add_tailable(file)
+        setup_file_tailer file
+        self
+      end
+
+      # Add a file created by opening +filename+ to this group after stepping
+      # +n+ lines backwards from the end of it.
+      def add_filename(filename, n = 0)
+        file = Logfile.open(filename.to_str, :backward => n)
+        file.backward n
+        setup_file_tailer file
+        self
+      end
+
+      # Iterate over all files contained in this group yielding to +block+ for
+      # each of them.
+      def each_file(&block)
+        each_tailer { |t| t.file }.map(&block)
+      end
+
+      # Iterate over all tailers in this group yielding to +block+ for each of
+      # them.
+      def each_tailer(&block)
+        @tailers.list.map(&block)
+      end
+
+      # Stop all tailers in this group at once.
+      def stop
+        each_tailer { |t| t.stop }
+        each_tailer { |t| t.join }
+        self
+      end
+
+      # Tail all the lines of all the files in the Tail::Group instance, that
+      # is yield to each of them.
+      #
+      # Every line is extended with the LineExtension module, that adds some
+      # methods to the line string. To get the path of the file this line was
+      # received from call line.file.path.
+      def tail
+        wait_for_activity do |tailer|
+          tailer.pending_lines.each do |line|
+            line.extend LineExtension
+            line.instance_variable_set :@tailer, tailer
+            yield line
+          end
+        end
+      end
+
+      private
+
+      def setup_file_tailer(file)
+        setup = ConditionVariable.new
+        mutex = Mutex.new
+        ft = nil
+        mutex.synchronize do
+          ft = Tailer.new do
+            t = Thread.current
+            t[:queue] = Queue.new
+            t[:file]  = file
+            mutex.synchronize do
+              setup.signal
+            end
+            file.tail { |line| t[:queue] << line }
+          end
+          setup.wait mutex
+        end
+        @tailers.add ft
+        nil
+      end
+
+      # Wait until new input is receіved on any of the tailers in the group. If
+      # so call +block+ with all of these trailers as an argument.
+      def wait_for_activity(&block)
+        loop do
+          pending = each_tailer.select(&:pending_lines?)
+          if pending.empty?
+            interval = each_file.map { |t| t.interval }.compact.min || 0.1
+            sleep interval
+          else
+            pending.each(&block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/io/tail/line_extension.rb

@@ -0,0 +1,15 @@
+class IO
+  module Tail
+    # This module is used to extend all lines received via one of the tailers
+    # of a File::Tail::Group.
+    module LineExtension
+      # The file as a File instance this line was read from.
+      def file
+        tailer.file
+      end
+
+      # This is the tailer this line was received from.
+      attr_reader :tailer
+    end
+  end
+end

data/lib/io/tail/logfile.rb

@@ -0,0 +1,85 @@
+class IO
+  module Tail
+    # This is an easy to use Logfile class that includes
+    # the File::Tail module.
+    #
+    # === Usage
+    # The unix command "tail -10f filename" can be emulated like that:
+    #  File::Tail::Logfile.open(filename, :backward => 10) do |log|
+    #    log.tail { |line| puts line }
+    #  end
+    #
+    # Or a bit shorter:
+    #  File::Tail::Logfile.tail(filename, :backward => 10) do |line|
+    #    puts line
+    #  end
+    #
+    # To skip the first 10 lines of the file do that:
+    #  File::Tail::Logfile.open(filename, :forward => 10) do |log|
+    #    log.tail { |line| puts line }
+    #  end
+    #
+    # The unix command "head -10 filename" can be emulated like that:
+    #  File::Tail::Logfile.open(filename, :return_if_eof => true) do |log|
+    #    log.tail(10) { |line| puts line }
+    #  end
+    class Logfile < File
+
+      # This method creates an File::Tail::Logfile object and
+      # yields to it, and closes it, if a block is given, otherwise it just
+      # returns it. The opts hash takes an option like
+      # * <code>:backward => 10</code> to go backwards
+      # * <code>:forward => 10</code> to go forwards
+      # in the logfile for 10 lines at the start. The buffersize
+      # for going backwards can be set with the
+      # * <code>:bufsiz => 8192</code> option.
+      # To define a callback, that will be called after a reopening occurs, use:
+      # * <code>:after_reopen => lambda { |file| p file }</code>
+      #
+      # Every attribute of File::Tail can be set with a <code>:attributename =>
+      # value</code> option.
+      def self.open(filename, opts = {}, &block) # :yields: file
+        file = new filename
+        opts.each do |o, v|
+          writer = o.to_s + "="
+          file.__send__(writer, v) if file.respond_to? writer
+        end
+        if opts.key?(:wind) or opts.key?(:rewind)
+          warn ":wind and :rewind options are deprecated, "\
+            "use :forward and :backward instead!"
+        end
+        if backward = opts[:backward] || opts[:rewind]
+          (args = []) << backward
+          args << opt[:bufsiz] if opts[:bufsiz]
+          file.backward(*args)
+        elsif forward = opts[:forward] || opts[:wind]
+          file.forward(forward)
+        end
+        if opts[:after_reopen]
+          file.after_reopen(&opts[:after_reopen])
+        end
+        if block_given?
+          begin
+            block.call file
+          ensure
+            file.close
+            nil
+          end
+        else
+          file
+        end
+      end
+
+      # Like open, but yields to every new line encountered in the logfile in
+      # +block+.
+      def self.tail(filename, opts = {}, &block)
+        if ([ :forward, :backward ] & opts.keys).empty?
+          opts[:backward] = 0
+        end
+        open(filename, opts) do |log|
+          log.tail { |line| block.call line }
+        end
+      end
+    end
+  end
+end

data/lib/io/tail/process.rb

@@ -0,0 +1,58 @@
+class IO
+  # This module can be included in your own File subclasses or used to extend
+  # files you want to tail.
+  module Tail
+    # A Process that's run and tailed
+    class Process  < IO::Tail::Tailable
+
+      attr_accessor :_command
+      attr_reader :_process
+
+      def initialize(command = nil)
+        super()
+        if command
+          @_command = command
+          self.reopen_tailable
+        end
+      end
+      # Taialble process should never have a EOF
+      # unless they are no longer tailable
+      def handle_EOFError
+        # Attempt to reopen
+        if @reopen_suspicious
+          raise ReopenException
+        end
+      end
+
+      # Ignore the mode
+      def reopen_tailable(mode = 'dummy')
+        @_process = IO.popen(@_command) if @_command
+      end
+      def readline
+        self._process.readline
+      end
+      # Used for testing purposes
+      def kill_inner
+        return if !self._process
+        killable = self._process.pid
+        $stdout.flush
+        begin
+          ::Process.kill 'INT', killable
+          ::Process.kill 'KILL', killable
+        rescue Exception => e
+          # Already killed ? Fine.
+        end
+      end
+      def close
+        return if !self._process
+        # We have to do that to stop the IO
+        self.kill_inner
+        begin
+          self._process.close
+        rescue Exception => e
+          # Ignore
+        end
+      end
+    end
+  end # module Tail
+end # class File

data/lib/io/tail/tailer.rb

@@ -0,0 +1,36 @@
+class IO
+  module Tail
+    # This class supervises activity on a tailed fail and collects newly read
+    # lines until the Tail::Group fetches and processes them.
+    class Tailer < ::Thread
+
+      # True if there are any lines pending on this Tailer, false otherwise.
+      def pending_lines?
+        !queue.empty?
+      end
+
+      # Fetch all the pending lines from this Tailer and thereby remove them
+      # from the Tailer's queue.
+      def pending_lines
+        Array.new(queue.size) { queue.deq(true) }
+      end
+
+      alias stop exit # Stop tailing this file and remove it from its File::Tail::Group.
+
+      # Return true if the thread local variable +id+ is defined or if this
+      # object responds to the method +id+.
+      def respond_to?(id)
+        !self[id].nil? || super
+      end
+
+      # Return the thread local variable +id+ if it is defined.
+      def method_missing(id, *args, &block)
+        if args.empty? && !(value = self[id]).nil?
+          value
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/io/tail/version.rb

@@ -0,0 +1,8 @@
+module IO::Tail
+  # IO::Tail version
+  VERSION         = '0.0.1'
+  VERSION_ARRAY   = VERSION.split(/\./).map { |x| x.to_i } # :nodoc:
+  VERSION_MAJOR   = VERSION_ARRAY[0] # :nodoc:
+  VERSION_MINOR   = VERSION_ARRAY[1] # :nodoc:
+  VERSION_BUILD   = VERSION_ARRAY[2] # :nodoc:
+end

data/tests/file_tail_group_test.rb

@@ -0,0 +1,86 @@
+#!/usr/bin/env ruby
+
+$: << File.dirname(__FILE__)
+$: << File.join(File.dirname(__FILE__),'..', 'lib')
+
+require 'test_helper'
+require 'io-tail'
+require 'timeout'
+require 'thread'
+require 'tempfile'
+Thread.abort_on_exception = true
+
+class FileTailGroupTest < Test::Unit::TestCase
+
+  def test_create_group
+    t, = make_file
+    g = IO::Tail::Group[t]
+    assert_equal t.path, g.each_tailer.first.file.path
+    assert_equal t.path, g.each_file.first.path
+  end
+
+  def test_stop_group
+    t, = make_file
+    g = IO::Tail::Group[t]
+    assert_equal t.path, g.each_tailer.first.file.path
+    assert_equal t.path, g.each_file.first.path
+    g.stop
+    assert_nil g.each_file.first
+  end
+
+  def test_add_file_to_group
+    g = IO::Tail::Group.new
+    t, = make_file
+    g.add_tailable t
+    assert_equal t.path, g.each_tailer.first.file.path
+    assert_equal t.path, g.each_file.first.path
+  end
+
+  def test_add_filename_to_group
+    g = IO::Tail::Group.new
+    t, name = make_file
+    t.close
+    g.add name
+    assert_equal name, g.each_tailer.first.file.path
+    assert_equal t.path, g.each_file.first.path
+  end
+
+  def test_add_generic_to_group
+    g = IO::Tail::Group.new
+    t1, n1 = make_file
+    t1.close
+    t2, n1 = make_file
+    g << n1
+    g << t2
+    assert g.each_tailer.any? { |t| t.file.path == n1 }
+    assert g.each_tailer.any? { |t| t.file.path == t2.path }
+    assert g.each_file.any? { |t| t.path == n1 }
+    assert g.each_file.any? { |t| t.path == t2.path }
+  end
+
+  def test_tail_multiple_files
+    t1, = make_file
+    t1.max_interval = 0.1
+    t2, = make_file
+    t2.max_interval = 0.1
+    g = IO::Tail::Group[t1, t2]
+    q = Queue.new
+    t = Thread.new do
+      g.tail { |l| q << l }
+    end
+    t1.puts "foo"
+    assert_equal "foo\n", q.pop
+    t2.puts "bar"
+    assert_equal "bar\n", q.pop
+  ensure
+    t and t.exit
+  end
+
+  private
+
+  def make_file
+    name = File.expand_path(::File.join(Dir.tmpdir, "tmp.#$$"))
+    file = File.open(name, 'w+')
+    return IO::Tail::File.new(file), name
+  end
+end

data/tests/file_tail_test.rb

@@ -0,0 +1,315 @@
+#!/usr/bin/env ruby
+
+$: << File.dirname(__FILE__)
+$: << File.join(File.dirname(__FILE__),'..', 'lib')
+
+require 'test_helper'
+require 'io-tail'
+require 'timeout'
+require 'thread'
+Thread.abort_on_exception = true
+
+class FileTailTest < Test::Unit::TestCase
+
+  def setup
+    @out = File.new("test.#$$", "wb")
+    append(@out, 100)
+    in_file = ::File.new(@out.path, "rb")
+    @in = IO::Tail::File.new(in_file)
+    @in.interval            = 0.4
+    @in.max_interval        = 0.8
+    @in.reopen_deleted      = true # is default
+    @in.reopen_suspicious   = true # is default
+    @in.suspicious_interval  = 60
+  end
+
+  def test_forward
+    [ 0, 1, 2, 10, 100 ].each do |lines|
+      @in.forward(lines)
+      assert_equal(100 - lines, count(@in))
+    end
+    @in.forward(101)
+    assert_equal(0, count(@in))
+  end
+
+  def test_backward
+    [ 0, 1, 2, 10, 100 ].each do |lines|
+      @in.backward(lines)
+      assert_equal(lines, count(@in))
+    end
+    @in.backward(101)
+    assert_equal(100, count(@in))
+  end
+
+  def test_backward_small_buffer
+    [ 0, 1, 2, 10, 100 ].each do |lines|
+      @in.backward(lines, 100)
+      assert_equal(lines, count(@in))
+    end
+    @in.backward(101, 100)
+    assert_equal(100, count(@in))
+  end
+
+  def test_backward_small_buffer2
+    @in.default_bufsize = 100
+    [ 0, 1, 2, 10, 100 ].each do |lines|
+      @in.backward(lines)
+      assert_equal(lines, count(@in))
+    end
+    @in.backward(101)
+    assert_equal(100, count(@in))
+  end
+
+  def test_tail_with_block_without_n
+    timeout(10) do
+      lines = []
+      @in.backward(1)
+      assert_raises(TimeoutError) do
+        timeout(1) { @in.tail { |l| lines << l } }
+      end
+      assert_equal(1, lines.size)
+      #
+      lines = []
+      @in.backward(10)
+      assert_raises(TimeoutError) do
+        timeout(1) { @in.tail { |l| lines << l } }
+      end
+      assert_equal(10, lines.size)
+      #
+      lines = []
+      @in.backward(100)
+      assert_raises(TimeoutError) do
+        timeout(1) { @in.tail { |l| lines << l } }
+      end
+      assert_equal(100, lines.size)
+      #
+      lines = []
+      @in.backward(101)
+      assert_raises(TimeoutError) do
+        timeout(1) { @in.tail { |l| lines << l } }
+      end
+    end
+  end
+
+  def test_tail_with_block_with_n
+    timeout(10) do
+      @in.backward(1)
+      lines = []
+      timeout(1) { @in.tail(0) { |l| lines << l } }
+      assert_equal(0, lines.size)
+      #
+      @in.backward(1)
+      lines = []
+      timeout(1) { @in.tail(1) { |l| lines << l } }
+      assert_equal(1, lines.size)
+      #
+      @in.backward(10)
+      lines = []
+      timeout(1) { @in.tail(10) { |l| lines << l } }
+      assert_equal(10, lines.size)
+      #
+      @in.backward(100)
+      lines = []
+      @in.backward(1)
+      assert_raises(TimeoutError) do
+        timeout(1) { @in.tail(2) { |l| lines << l } }
+      end
+      assert_equal(1, lines.size)
+      #
+    end
+  end
+
+  def test_tail_without_block_with_n
+    timeout(10) do
+      @in.backward(1)
+      lines = []
+      timeout(1) { lines += @in.tail(0) }
+      assert_equal(0, lines.size)
+      #
+      @in.backward(1)
+      lines = []
+      timeout(1) { lines += @in.tail(1) }
+      assert_equal(1, lines.size)
+      #
+      @in.backward(10)
+      lines = []
+      timeout(1) { lines += @in.tail(10) }
+      assert_equal(10, lines.size)
+      #
+      @in.backward(100)
+      lines = []
+      @in.backward(1)
+      assert_raises(TimeoutError) do
+        timeout(1) { lines += @in.tail(2) }
+      end
+      assert_equal(0, lines.size)
+    end
+  end
+
+  def test_tail_withappend
+    @in.backward
+    lines = []
+    logger = Thread.new do
+      begin
+        timeout(1) { @in.tail { |l| lines << l } }
+      rescue TimeoutError
+      end
+    end
+    appender = Thread.new { append(@out, 10) }
+    appender.join
+    logger.join
+    assert_equal(10, lines.size)
+  end
+
+  def test_tail_truncated
+    @in.backward
+    lines = []
+    logger = Thread.new do
+      begin
+        timeout(1) do
+          @in.tail do |l|
+            lines << l
+          end
+        end
+      rescue TimeoutError
+      end
+    end
+    appender = Thread.new do
+      until logger.stop?
+        sleep 0.1
+      end
+      @out.close
+      File.truncate(@out.path, 0)
+      @out = File.new(@in._file.path, "ab")
+      append(@out, 10)
+    end
+    appender.join
+    logger.join
+    assert_equal(10, lines.size)
+  end
+
+  def test_tail_remove
+    return if File::PATH_SEPARATOR == ';' # Grmpf! Windows...
+    @in.backward
+    reopened = false
+    @in.after_reopen { |f| reopened = true }
+    lines = []
+    logger = Thread.new do
+      begin
+        timeout(2) do
+          @in.tail do |l|
+            lines << l
+          end
+        end
+      rescue TimeoutError
+      end
+    end
+    appender = Thread.new do
+      until logger.stop?
+        sleep 0.1
+      end
+      @out.close
+      File.unlink(@out.path)
+      @out = File.new(@in._file.path, "wb")
+      append(@out, 10)
+    end
+    appender.join
+    logger.join
+    assert_equal(10, lines.size)
+    assert reopened
+  end
+
+  def test_tail_remove2
+    return if File::PATH_SEPARATOR == ';' # Grmpf! Windows...
+    @in.backward
+    reopened = false
+    @in.after_reopen { |f| reopened = true }
+    lines = []
+    logger = Thread.new do
+      begin
+        timeout(2) do
+          @in.tail do |l|
+            lines << l
+          end
+        end
+      rescue TimeoutError
+      end
+    end
+    appender = Thread.new do
+      until logger.stop?
+        sleep 0.1
+      end
+      @out.close
+      File.unlink(@out.path)
+      @out = File.new(@in._file.path, "wb")
+      append(@out, 10)
+      sleep 1
+      append(@out, 10)
+      File.unlink(@out.path)
+      @out = File.new(@in._file.path, "wb")
+      append(@out, 10)
+    end
+    appender.join
+    logger.join
+    assert_equal(30, lines.size)
+    assert reopened
+  end
+
+  def test_tail_remove3
+    return if File::PATH_SEPARATOR == ';' # Grmpf! Windows...
+    @in.backward
+    reopened = false
+    @in.after_reopen { |f| reopened = true }
+    lines = []
+    logger = Thread.new do
+      begin
+        timeout(2) do
+          @in.tail(15) do |l|
+            lines << l
+          end
+        end
+      rescue TimeoutError
+      end
+    end
+    appender = Thread.new do
+      until logger.stop?
+        sleep 0.1
+      end
+      @out.close
+      File.unlink(@out.path)
+      @out = File.new(@in._file.path, "wb")
+      append(@out, 10)
+      sleep 1
+      append(@out, 10)
+      File.unlink(@out.path)
+      @out = File.new(@in._file.path, "wb")
+      append(@out, 10)
+    end
+    appender.join
+    logger.join
+    assert_equal(15, lines.size)
+    assert reopened
+  end
+
+  def teardown
+    @in.close
+    @out.close
+    File.unlink(@out.path)
+  end
+
+  private
+
+  def count(file)
+    n = 0
+    until file._file.eof?
+      file.readline
+      n += 1
+    end
+    return n
+  end
+
+  def append(file, n, size = 70)
+    (1..n).each { |x| file << "#{x} #{"A" * size}\n" }
+    file.flush
+  end
+end