franz 1.2.7

data/lib/franz/sash.rb

@@ -0,0 +1,81 @@
+require 'thread'
+
+module Franz
+
+  # Sash - A threadsafe hash/array hybrid with access times
+  #
+  # @example
+  #   s = Sash.new           # => #<Sash...>
+  #   s.keys                 # => []
+  #   s.insert :key, :value  # => value
+  #   s.get :key             # => [:value]
+  #   s.insert :key, :crazy  # => :crazy
+  #   s.mtime :key           # => 2014-02-18 21:24:30 -0800
+  #   s.flush :key           # => [:value, :crazy]
+  #
+  # Think of it like a Hash where the keys map to "value buffers"
+  class Sash
+
+    # Create a new, empty Sash.
+    def initialize
+      @mutex = Mutex.new
+      @mtime = Hash.new { |default, key| default[key] = nil }
+      @hash  = Hash.new { |default, key| default[key] = []  }
+    end
+
+    # Grab a list of known keys.
+    #
+    # @return [Array<Object>]
+    def keys ; @hash.keys end
+
+    # Insert a value into a key's value buffer.
+    #
+    # @param key [Object]
+    # @param value [Object]
+    #
+    # @return [Object] the value
+    def insert key, value
+      @mutex.synchronize do
+        @hash[key] << value
+        @mtime[key] = Time.now
+      end
+      return value
+    end
+
+    # Return a key's value buffer.
+    #
+    # @param [Object] key
+    #
+    # @return [Array<Object>]
+    def get key ; @hash[key] end
+
+    # Remove and return a key's value buffer.
+    #
+    # @param [Object] key
+    #
+    # @return [Array<Object>]
+    def remove key ; @hash.delete(key) end
+
+    # Return the last time the key's value buffer was modified.
+    #
+    # @param [Object] key
+    #
+    # @return [Time]
+    def mtime key ; @mtime[key] end
+
+    # Flush and return a key's value buffer.
+    #
+    # @param [Object] key
+    #
+    # @return [Array<Object>]
+    def flush key
+      value = nil
+      @mutex.synchronize do
+        value       = @hash[key]
+        @hash[key]  = []
+        @mtime[key] = Time.now
+      end
+      return value
+    end
+  end
+end

data/lib/franz/tail.rb

@@ -0,0 +1,191 @@
+require 'thread'
+require 'logger'
+
+require 'buftok'
+
+module Franz
+
+  # Tail receives low-level file events from a Watch and handles the actual
+  # reading of files, providing a stream of lines.
+  class Tail
+    attr_reader :cursors
+
+    # Start a new Tail thread in the background.
+    #
+    # @param opts [Hash] a complex Hash for tail configuration
+    def initialize opts={}
+      @watch_events      = opts[:watch_events]      || []
+      @tail_events       = opts[:tail_events]       || []
+      @eviction_interval = opts[:eviction_interval] || 60
+      @block_size        = opts[:block_size]        || 32_768 # 32 KiB
+      @spread_size       = opts[:spread_size]       || 98_304 # 96 KiB
+      @cursors           = opts[:cursors]           || Hash.new
+      @logger            = opts[:logger]            || Logger.new(STDOUT)
+
+      @buffer  = Hash.new { |h, k| h[k] = BufferedTokenizer.new }
+      @file    = Hash.new
+      @changed = Hash.new
+      @reading = Hash.new
+      @stop    = false
+
+      @evict_thread = Thread.new do
+        log.debug 'starting tail-evict'
+        until @stop
+          evict
+          sleep eviction_interval
+        end
+        sleep eviction_interval
+        evict
+      end
+
+      @backlog  = Hash.new { |h, k| h[k] = Array.new }
+      @incoming = Hash.new { |h, k| h[k] = SizedQueue.new 10_000 }
+
+      @watch_thread = Thread.new do
+        log.debug 'starting tail-watch'
+        until @stop
+          e = watch_events.shift
+          @incoming[e[:path]].push e
+        end
+      end
+
+      @tail_thread = Thread.new do
+        until @stop
+          had_event = false
+
+          paths = (@backlog.keys + @incoming.keys).uniq.shuffle
+
+          paths.each do |path|
+            event = @backlog[path].shift
+            begin
+              event = @incoming[path].shift(true)
+            rescue ThreadError
+              next
+            end if event.nil?
+            had_event = true
+            handle event
+          end
+
+          sleep 0.05 unless had_event
+        end
+      end
+
+      log.debug 'started tail'
+    end
+
+    # Stop the Tail thread. Effectively only once.
+    #
+    # @return [Hash] internal "cursors" state
+    def stop
+      return state if @stop
+      @stop = true
+      @watch_thread.kill
+      @evict_thread.join
+      @tail_thread.join
+      log.debug 'stopped tail'
+      return state
+    end
+
+    # Return the internal "cursors" state
+    def state
+      return @cursors.dup
+    end
+
+  private
+    attr_reader :watch_events, :tail_events, :eviction_interval, :block_size, :cursors, :file, :buffer, :changed, :reading
+
+    def log ; @logger end
+
+    def open path
+      return true unless file[path].nil?
+      pos = @cursors.include?(path) ? @cursors[path] : 0
+      begin
+        file[path] = File.open(path)
+        file[path].sysseek pos, IO::SEEK_SET
+        @cursors[path] = pos
+        @changed[path] = Time.now.to_i
+      rescue Errno::ENOENT
+        return false
+      end
+      log.debug 'opened: path=%s' % path.inspect
+      return true
+    end
+
+    def read path, size
+      @reading[path] = true
+
+      bytes_read = 0
+      loop do
+        begin
+          break if file[path].pos >= size
+        rescue NoMethodError
+          break unless open(path)
+          break if file[path].pos >= size
+        end
+
+        if bytes_read >= @spread_size
+          @backlog[path].push name: :appended, path: path, size: size
+          break
+        end
+
+        begin
+          data = file[path].sysread @block_size
+          buffer[path].extract(data).each do |line|
+            log.trace 'captured: path=%s line=%s' % [ path, line ]
+            tail_events.push path: path, line: line
+          end
+        rescue EOFError, Errno::ENOENT
+          # we're done here
+        end
+
+        last_pos = @cursors[path]
+        @cursors[path] = file[path].pos
+        bytes_read += @cursors[path] - last_pos
+      end
+
+      log.trace 'read: path=%s size=%s' % [ path.inspect, size.inspect ]
+      @changed[path] = Time.now.to_i
+      @reading.delete path
+    end
+
+    def close path
+      @reading[path] = true # prevent evict from interrupting
+      file.delete(path).close if file.include? path
+      @cursors.delete(path)
+      @changed.delete(path)
+      @reading.delete(path)
+      log.debug 'closed: path=%s' % path.inspect
+    end
+
+    def evict
+      file.keys.each do |path|
+        next if @reading[path]
+        next unless @changed[path] < Time.now.to_i - eviction_interval
+        next unless file.include? path
+        next unless @incoming[path].empty?
+        next unless @backlog[path].empty?
+        file.delete(path).close
+        log.debug 'evicted: path=%s' % path.inspect
+      end
+    end
+
+    def handle event
+      log.trace 'handle: event=%s' % event.inspect
+      case event[:name]
+      when :created
+      when :replaced
+        close event[:path]
+        read event[:path], event[:size]
+      when :truncated
+        close event[:path]
+        read event[:path], event[:size]
+      when :appended
+        read event[:path], event[:size]
+      when :deleted
+        close event[:path]
+      else
+        raise 'invalid event'
+      end
+    end
+  end
+end

data/lib/franz/tail_pool.rb

@@ -0,0 +1,68 @@
+require 'thread'
+require 'logger'
+
+require 'consistent_hashing'
+
+module Franz
+
+  # TailPool creates a consistenly-hashed pool of Tails.
+  class TailPool
+    # Start a new TailPool thread in the background.
+    #
+    # @param opts [Hash] a complex Hash for configuration
+    def initialize opts={}
+      @size         = opts[:size]         || 5
+      @watch_events = opts[:watch_events] || []
+      @tail_events  = opts[:tail_events]  || []
+      @size         = opts[:size]         || 5
+      @logger       = opts[:logger]       || Logger.new(STDOUT)
+
+      @tails  = []
+      @ring   = ConsistentHashing::Ring.new
+      @events = Hash.new { |h, k| h[k] = SizedQueue.new 10_000 }
+
+      @size.times do |i|
+        log.debug 'starting tail_pool-tail #%d' %  i
+        @ring << @events[i]
+        @tails << Franz::Tail.new(opts.merge({
+          watch_events: @events[i],
+          tail_events: @tail_events
+        }))
+      end
+
+      @stop = false
+
+      @in_thread = Thread.new do
+        log.debug 'starting tail_pool-watch'
+        until @stop
+          e = @watch_events.shift
+          q = @ring.node_for e[:path]
+          q.push e
+        end
+      end
+
+      log.debug 'started tail_pool'
+    end
+
+    # Stop the TailPool thread. Effectively only once.
+    #
+    # @return [Hash] internal "cursors" state
+    def stop
+      return state if @stop
+      @stop = true
+      @tails.map(&:stop)
+      log.debug 'stopped tail_pool'
+      return state
+    end
+
+    # Return the internal "cursors" state
+    def state
+      @tails.map(&:state).reduce(&:merge)
+    end
+
+  private
+    attr_reader :watch_events, :tail_events, :size
+
+    def log ; @logger end
+  end
+end

data/lib/franz/watch.rb

@@ -0,0 +1,180 @@
+require 'logger'
+
+module Franz
+
+  # Watch works in tandem with Discover to maintain a list of known files and
+  # their status. Events are generated when a file is created, destroyed, or
+  # modified (including appended, truncated, and replaced).
+  class Watch
+
+    # Start a new Watch thread in the background.
+    #
+    # @param [Hash] opts options for the watch
+    # @option opts [Queue] :discoveries ([]) "input" queue of discovered paths
+    # @option opts [Queue] :deletions ([]) "output" queue of deleted paths
+    # @option opts [Queue] :watch_events ([]) "output" queue of file events
+    # @option opts [Fixnum] :watch_interval (1) seconds between watch rounds
+    # @option opts [Hash<Path,State>] :stats ([]) internal "stats" state
+    # @option opts [Logger] :logger (Logger.new(STDOUT)) logger to use
+    def initialize opts={}
+      @discoveries    = opts[:discoveries]    || []
+      @deletions      = opts[:deletions]      || []
+      @watch_events   = opts[:watch_events]   || []
+      @watch_interval = opts[:watch_interval] || 10
+      @stats          = opts[:stats]          || Hash.new
+      @logger         = opts[:logger]         || Logger.new(STDOUT)
+
+      # Need to resend old events to make sure Tail catches up
+      stats.each do |path, old_stat|
+        watch_events.push name: :appended, path: path, size: old_stat[:size]
+      end
+
+      @stop = false
+
+      @thread = Thread.new do
+        log.debug 'starting watch-discover'
+        until @stop
+          until discoveries.empty?
+            d = discoveries.pop
+            @stats[d] = nil
+          end
+          watch.each do |deleted|
+            @stats.delete deleted
+            deletions.push deleted
+          end
+          sleep watch_interval
+        end
+      end
+
+      log.debug 'started watch'
+    end
+
+    # Stop the Watch thread. Effectively only once.
+    #
+    # @return [Hash] internal "stats" state
+    def stop
+      return state if @stop
+      @stop = true
+      @thread.join
+      log.debug 'stopped watch'
+      return state
+    end
+
+    # Return the internal "stats" state
+    def state
+      return @stats.dup
+    end
+
+  private
+    attr_reader :discoveries, :deletions, :watch_events, :watch_interval, :stats
+
+    def log ; @logger end
+
+    def enqueue name, path, size=nil
+      log.trace 'enqueue: name=%s path=%s size=%s' % [
+        name.inspect, path.inspect, size.inspect
+      ]
+      watch_events.push name: name, path: path, size: size
+    end
+
+    def watch
+      deleted = []
+      stats.keys.each do |path|
+        old_stat    = stats[path]
+        stat        = stat_for path
+        stats[path] = stat
+
+        if file_created? old_stat, stat
+          enqueue :created, path
+        elsif file_deleted? old_stat, stat
+          enqueue :deleted, path
+          deleted << path
+        end
+
+        if file_replaced? old_stat, stat
+          enqueue :replaced, path, stat[:size]
+        elsif file_appended? old_stat, stat
+          enqueue :appended, path, stat[:size]
+        elsif file_truncated? old_stat, stat
+          enqueue :truncated, path, stat[:size]
+        end
+      end
+      return deleted
+    end
+
+
+
+    # Perform a file stat and return a simplified version.
+    #
+    # @param path [String] file path to examine
+    def stat_for path
+      return begin
+        stat = File::Stat.new(path)
+        {
+          inode: {
+            ino: stat.ino,
+            maj: stat.dev_major,
+            min: stat.dev_minor
+          },
+          size: stat.size
+        }
+      rescue Errno::ENOENT
+        nil
+      end
+    end
+
+    # Grab only the inode from a stat (or nil if the stat is nil).
+    #
+    # @param stat [Stat] stat to inspect
+    def inode_for stat
+      return nil if stat.nil?
+      return stat[:inode].to_a
+    end
+
+    # Detect whether the file was created.
+    #
+    # @param old_stat [Stat] stat before some change
+    # @param new_stat [Stat] stat after some change
+    def file_created? old_stat, new_stat
+      return !new_stat.nil? && old_stat.nil?
+    end
+
+    # Detect whether the file was deleted.
+    #
+    # @param old_stat [Stat] stat before some change
+    # @param new_stat [Stat] stat after some change
+    def file_deleted? old_stat, new_stat
+      return new_stat.nil? && !old_stat.nil?
+    end
+
+    # Detect whether the file was replaced (e.g. inode changed).
+    #
+    # @param old_stat [Stat] stat before some change
+    # @param new_stat [Stat] stat after some change
+    def file_replaced? old_stat, new_stat
+      return false if new_stat.nil?
+      return false if old_stat.nil?
+      return inode_for(new_stat) != inode_for(old_stat)
+    end
+
+    # Detect whether the file was truncated (e.g. rotated).
+    #
+    # @param old_stat [Stat] stat before some change
+    # @param new_stat [Stat] stat after some change
+    def file_truncated? old_stat, new_stat
+      return false if new_stat.nil?
+      return false if old_stat.nil?
+      return new_stat[:size] < old_stat[:size]
+    end
+
+    # Detect whether the file was appended.
+    #
+    # @param old_stat [Stat] stat before some change
+    # @param new_stat [Stat] stat after some change
+    def file_appended? old_stat, new_stat
+      return false if new_stat.nil?
+      return new_stat[:size] > 0 if old_stat.nil?
+      return new_stat[:size] > old_stat[:size]
+    end
+  end
+end