franz 1.3.1 → 1.4.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2d9eacee0d34d4d4f967ef3bbf4a0ccd6b587d05
-  data.tar.gz: 2e6121256b03b8a4d851faba6e3435fcb565035f
+  metadata.gz: 6ee78d80face7d2a796de3551493f5587300b236
+  data.tar.gz: 573935e9612b3c002f3430d9d19d6374e6354e71
 SHA512:
-  metadata.gz: 878fe9d21753ed5de42a5ffd21104f47b34f3c0a83d3fd499b2265a9e3f227ec77fba87b18bc9396c557cc08b441c129bf12187218e3d8665ec7facbf34b043d
-  data.tar.gz: 721f29b53bdcecdb1cabf8c0c6c142c318857671e241e05bd858c00ab86d6be0de97fb25dde83ec6c1833de6ff5ba7b0a2edea0f31ccad7f4312952c9571d998
+  metadata.gz: 8d9eb1d74c5f3aafe0c05324a66831ae578936f7fd4aad758a792329c77286ccef990590669356ecc95364245c5319ce2cf76edcdc4ea81f8f3a675d5a126071
+  data.tar.gz: bf437303886139f402244f4161075341564ad36d161df59fc58f98f834bf50a59ac67a4fa55a14e42047bbb697eb835be4f24b48d68724ae3492c37b33732e6d

data/Readme.md

@@ -34,13 +34,13 @@ But it's not yet officially sanctioned. Such is life. At any rate, you don't
 have to deal with this issue in Franz, he flushes inactive buffers after a time.
 Easy-peasy, lemon-squeezy.
 
-### File Handle Eviction
+### File Hande-ing
 
 Now I'm not actually sure this issue affects logstash proper, but it's one you
 might face if you decide to write your own, so here goes: If you're tailing a
 bunch of files and you never let go of their file handles, you might very well
 exhaust your ulimit after running for a while. Because Franz is designed to be
-a daemon, he releases or "evicts" file handles after a period of inactivity.
+a daemon, he only opens file handles when necessary.
 
 ### Sequential Identifiers
 

data/VERSION

@@ -1 +1 @@
-1.3.1
+1.4.14

data/bin/franz

@@ -45,37 +45,43 @@ config = Franz::Config.new opts[:config]
 logger = Franz::Logger.new opts[:debug], opts[:trace], opts[:log]
 
 io_bound = config[:output][:bound] || 10_000
+io = SizedQueue.new io_bound
 
-# begin
-  io = SizedQueue.new io_bound
+# Now we'll connect to our output, RabbitMQ. This creates a new thread in the
+# background, which will consume the events generated by our input on io
+Franz::Output.new \
+  input: io,
+  output: config[:output][:rabbitmq],
+  logger: logger,
+  tags: config[:output][:tags]
 
-  # Now we'll connect to our output, RabbitMQ. This creates a new thread in the
-  # background, which will consume the events generated by our input on io
-  fout = Franz::Output.new \
-    input: io,
-    output: config[:output][:rabbitmq],
-    logger: logger,
-    tags: config[:output][:tags]
+# Franz has only one kind of input, plain text files.
+Franz::Input.new \
+  input: config[:input],
+  output: io,
+  logger: logger,
+  checkpoint: config[:checkpoint],
+  checkpoint_interval: config[:checkpoint_interval]
 
-  # Franz has only one kind of input, plain text files.
-  fin = Franz::Input.new \
-    input: config[:input],
-    output: io,
-    logger: logger,
-    checkpoint: config[:checkpoint],
-    checkpoint_interval: config[:checkpoint_interval]
+# Ensure memory doesn't grow too large (> 1GB by default)
+def mem_kb ; `ps -o rss= -p #{$$}`.strip.to_i ; end
 
-  # Remember, both the input and output were started up in background threads,
-  # so we'll have to wait here in main or else we'll just exit.
-  fout.join
+mem_limit = config[:memory_limit] || 1_000_000
+mem_sleep = config[:memory_limit_interval] || 60
 
-# rescue SignalException => e
-#   logger.fatal "#{e.inspect} #{$!}\n\t#{$@ * "\n\t"}"
-# rescue SystemExit, Interrupt => e
-#   logger.fatal "#{e.inspect} #{$!}\n\t#{$@ * "\n\t"}"
-# ensure
-#   logger.info 'Draining. This may take a while...'
-#   fin.stop
-#   fin.checkpoint
-#   logger.info 'Bye!'
-# end
+loop do
+  sleep mem_sleep
+  mem_used = mem_kb
+  if mem_used > mem_limit
+    logger.fatal \
+      event: 'killed',
+      reason: 'Consuming too much memory',
+      used: mem_used,
+      limit: mem_limit
+    exit(1)
+  end
+  logger.debug \
+    event: 'memcheck',
+    used: mem_used,
+    limit: mem_limit
+end

data/franz.gemspec

@@ -12,11 +12,10 @@ Gem::Specification.new do |s|
   s.description = Franz::SUMMARY + '.'
 
   s.add_runtime_dependency 'bunny', '~> 1'
-  s.add_runtime_dependency 'buftok', '~> 0'
   s.add_runtime_dependency 'trollop', '~> 2'
   s.add_runtime_dependency 'colorize', '~> 0'
   s.add_runtime_dependency 'deep_merge', '~> 1'
-  s.add_runtime_dependency 'consistent-hashing', '~> 1'
+  s.add_runtime_dependency 'eventmachine', '~> 1'
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- test/*`.split("\n")

data/lib/franz/agg.rb

@@ -29,18 +29,17 @@ module Franz
       @tail_events = opts[:tail_events] || []
       @agg_events  = opts[:agg_events]  || []
 
+      @buffer_limit   = opts[:buffer_limit]   || 50
       @flush_interval = opts[:flush_interval] || 10
       @seqs           = opts[:seqs]           || Hash.new
       @logger         = opts[:logger]         || Logger.new(STDOUT)
 
       @types  = Hash.new
-      @lock   = Mutex.new
+      @lock   = Hash.new { |h,k| h[k] = Mutex.new }
       @buffer = Franz::Sash.new
       @stop   = false
 
-      log.debug 'agg: configs=%s tail_events=%s agg_events=%s' % [
-        @configs, @tail_events, @agg_events
-      ]
+      @num_events = 0
 
       @t1 = Thread.new do
         until @stop
@@ -54,7 +53,11 @@ module Franz
         capture until @stop
       end
 
-      log.debug 'started agg'
+      log.debug \
+        event: 'agg started',
+        configs: @configs,
+        tail_events: @tail_events,
+        agg_events: @agg_events
     end
 
     # Stop the Agg thread. Effectively only once.
@@ -65,7 +68,7 @@ module Franz
       @stop = true
       @t2.kill
       @t1.join
-      log.debug 'stopped agg'
+      log.debug event: 'agg stopped'
       return state
     end
 
@@ -92,45 +95,84 @@ module Franz
             }
             included && !excluded
           }
-          return @types[path] = type unless type.nil?
+          unless type.nil?
+            @types[path] = type
+            return type
+          end
         end
-        log.error 'Could not identify type for path=%s' % path
+        log.warn \
+          event: 'type unknown',
+          path: path
+        @types[path] = nil
+        return nil
       end
     end
 
     def config path
-      configs.select { |c| c[:type] == type(path) }.shift
+      t = type(path)
+      configs.select { |c| c[:type] == t }.shift
     end
 
     def seq path
       seqs[path] = seqs.fetch(path, 0) + 1
     end
 
-    def real_path path
-      Pathname.new(path).realpath.to_s rescue path
+    def drop? path, message
+      drop = config(path)[:drop]
+      if drop
+        drop = drop.is_a?(Array) ? drop : [ drop ]
+        drop.each do |pattern|
+          return true if message =~ pattern
+        end
+      end
+      return false
     end
 
     def enqueue path, message
-      p = real_path path
+      if drop? path, message
+        log.trace \
+          event: 'dropped',
+          path: path,
+          message: message
+        return
+      end
+
       t = type path
+      if t.nil?
+        log.trace \
+          event: 'enqueue skipped',
+          path: path,
+          message: message
+        return
+      end
+
+      log.trace \
+        event: 'enqueue',
+        path: path,
+        message: message
       s = seq path
       m = message.encode 'UTF-8', invalid: :replace, undef: :replace, replace: '?'
-      log.trace 'enqueue type=%s path=%s seq=%d message=%s' % [
-        t.inspect, p.inspect, s.inspect, m.inspect
-      ]
-      agg_events.push path: p, message: m, type: t, host: @@host, '@seq' => s
+      agg_events.push path: path, message: m, type: t, host: @@host, '@seq' => s
     end
 
     def capture
       event = tail_events.shift
-      log.trace 'received path=%s line=%s' % [
-        event[:path], event[:line]
-      ]
-      multiline = config(event[:path])[:multiline]
+      log.trace \
+        event: 'capture',
+        raw: event
+      multiline = config(event[:path])[:multiline] rescue nil
       if multiline.nil?
         enqueue event[:path], event[:line] unless event[:line].empty?
       else
-        lock.synchronize do
+        lock[event[:path]].synchronize do
+          size = buffer.size(event[:path])
+          if size > @buffer_limit
+            log.trace \
+              event: 'buffer overflow',
+              path: event[:path],
+              size: size,
+              limmit: @buffer_limit
+          end
           if event[:line] =~ multiline
             buffered = buffer.flush(event[:path])
             lines    = buffered.map { |e| e[:line] }.join("\n")
@@ -141,12 +183,14 @@ module Franz
       end
     end
 
-    def flush force=false
-      lock.synchronize do
-        started = Time.now
-        buffer.keys.each do |path|
-          if started - buffer.mtime(path) >= flush_interval || force
-            log.trace 'flushing path=%s' % path.inspect
+    def flush force=false, started=Time.now
+      log.debug \
+        event: 'flush',
+        force: force,
+        started: started
+      buffer.keys.each do |path|
+        lock[path].synchronize do
+          if force || started - buffer.mtime(path) >= flush_interval
             buffered = buffer.remove(path)
             lines    = buffered.map { |e| e[:line] }.join("\n")
             enqueue path, lines unless lines.empty?

data/lib/franz/discover.rb

@@ -1,4 +1,6 @@
+require 'set'
 require 'logger'
+require 'shellwords'
 
 
 # Discover performs half of file existence detection by expanding globs and
@@ -25,35 +27,45 @@ class Franz::Discover
     @known             = opts[:known]             || []
     @logger            = opts[:logger]            || Logger.new(STDOUT)
 
+    @known = Set.new(@known)
+
     @configs = configs.map do |config|
       config[:includes] ||= []
       config[:excludes] ||= []
       config
     end
 
-    @stop = false
 
-    log.debug 'discover: configs=%s discoveries=%s deletions=%s' % [
-      @configs, @discoveries, @deletions
-    ]
+    @stop = false
 
     @thread = Thread.new do
       until @stop
         until deletions.empty?
           d = deletions.pop
           @known.delete d
-          log.debug 'deleted: %s' % d.inspect
+          log.debug \
+            event: 'discover deleted',
+            path: d
         end
+
         discover.each do |discovery|
           discoveries.push discovery
-          @known.push discovery
-          log.debug 'discovered: %s' % discovery.inspect
+          @known.add discovery
+          log.debug \
+            event: 'discover discovered',
+            path: discovery
         end
         sleep discover_interval
       end
     end
 
-    log.debug 'started discover'
+    log.debug \
+      event: 'discover started',
+      configs: configs,
+      discoveries: discoveries,
+      deletions: deletions,
+      discover_interval: discover_interval,
+      ignore_before: ignore_before
   end
 
   # Stop the Discover thread. Effectively only once.
@@ -63,51 +75,34 @@ class Franz::Discover
     return state if @stop
     @stop = true
     @thread.kill
-    log.debug 'stopped discover'
+    log.debug event: 'discover stopped'
     return state
   end
 
   # Return the internal "known" state
   def state
-    return @known.dup
+    return @known.to_a
   end
 
 private
-  attr_reader :configs, :discoveries, :deletions, :discover_interval, :known
+  attr_reader :configs, :discoveries, :deletions, :discover_interval, :known, :ignore_before
 
   def log ; @logger end
 
   def discover
+    log.debug event: 'discover'
     discovered = []
     configs.each do |config|
       config[:includes].each do |glob|
-        expand(glob).each do |path|
+        Dir[glob].each do |path|
+          next if known.include? path
           next if config[:excludes].any? { |exclude|
             File.fnmatch? exclude, File::basename(path)
           }
-          next if known.include? path
-          next unless File.file? path
-          next if File.mtime(path).to_i <= @ignore_before
           discovered.push path
         end
       end
     end
     return discovered
   end
-
-  def expand glob
-    Dir[glob]
-    # dir_glob = File.dirname(glob)
-    # file_glob = File.basename(glob)
-    # files = []
-    # Dir.glob(dir_glob).each do |dir|
-    #   next unless File::directory?(dir)
-    #   Dir.foreach(dir) do |fname|
-    #     next if fname == '.' || fname == '..'
-    #     next unless File.fnmatch?(file_glob, fname)
-    #     files << File.join(dir, fname)
-    #   end
-    # end
-    # files
-  end
 end

data/lib/franz/input.rb

@@ -37,6 +37,11 @@ module Franz
           watch_interval: nil,
           eviction_interval: nil,
           flush_interval: nil,
+          buffer_limit: nil,
+          line_limit: nil,
+          play_catchup?: nil,
+          skip_interval: nil,
+          stale_interval: nil,
           configs: []
         }
       }.deep_merge!(opts)
@@ -47,8 +52,6 @@ module Franz
       @checkpoint_path     = opts[:checkpoint].sub('*', '%d')
       @checkpoint_glob     = opts[:checkpoint]
 
-      log.debug 'input: opts=%s' % JSON::pretty_generate(opts)
-
       # The checkpoint contains a Marshalled Hash with a compact representation of
       # stateful inputs to various Franz streaming classes (e.g. the "known" option
       # to Franz::Discover). This state file is generated automatically every time
@@ -60,9 +63,13 @@ module Franz
       unless last_checkpoint_path.nil?
         last_checkpoint = File.read(last_checkpoint_path)
         state = Marshal.load last_checkpoint
-        log.debug 'Loaded %s' % last_checkpoint_path.inspect
+        log.info \
+          event: 'input checkpoint loaded',
+          checkpoint: last_checkpoint_path
       end
 
+      full_state = state.nil? ? nil : state.dup
+
       state = state || {}
       known = state.keys
       stats, cursors, seqs = {}, {}, {}
@@ -74,14 +81,11 @@ module Franz
         stats[path]   = state[path]
       end
 
-      log.debug 'starting input...'
-
       discoveries  = SizedQueue.new opts[:input][:discover_bound]
       deletions    = SizedQueue.new opts[:input][:discover_bound]
       watch_events = SizedQueue.new opts[:input][:watch_bound]
       tail_events  = SizedQueue.new opts[:input][:tail_bound]
 
-      log.debug 'starting discover...'
       @disover = Franz::Discover.new \
         discoveries: discoveries,
         deletions: deletions,
@@ -89,44 +93,49 @@ module Franz
         discover_interval: opts[:input][:discover_interval],
         ignore_before: opts[:input][:ignore_before],
         logger: opts[:logger],
-        known: known
+        known: known,
+        full_state: full_state
+
+      @watch = Franz::Watch.new \
+        discoveries: discoveries,
+        deletions: deletions,
+        watch_events: watch_events,
+        watch_interval: opts[:input][:watch_interval],
+        play_catchup?: opts[:input][:play_catchup?],
+        skip_interval: opts[:input][:skip_interval],
+        stale_interval: opts[:input][:stale_interval],
+        logger: opts[:logger],
+        stats: stats,
+        full_state: full_state
 
-      log.debug 'starting tail...'
       @tail = Franz::Tail.new \
         watch_events: watch_events,
         tail_events: tail_events,
-        eviction_interval: opts[:input][:eviction_interval],
+        block_size: opts[:input][:block_size],
+        line_limit: opts[:input][:line_limit],
         logger: opts[:logger],
-        cursors: cursors
+        cursors: cursors,
+        full_state: full_state
 
-      log.debug 'starting agg...'
       @agg = Franz::Agg.new \
         configs: opts[:input][:configs],
         tail_events: tail_events,
         agg_events: opts[:output],
         flush_interval: opts[:input][:flush_interval],
+        buffer_limit: opts[:input][:buffer_limit],
         logger: opts[:logger],
-        seqs: seqs
-
-      log.debug 'starting watch...'
-      @watch = Franz::Watch.new \
-        discoveries: discoveries,
-        deletions: deletions,
-        watch_events: watch_events,
-        watch_interval: opts[:input][:watch_interval],
-        logger: opts[:logger],
-        stats: stats
+        seqs: seqs,
+        full_state: full_state
 
       @stop = false
       @t = Thread.new do
-        log.debug 'starting checkpoint'
         until @stop
           checkpoint
           sleep @checkpoint_interval
         end
       end
 
-      log.debug 'started input'
+      log.info event: 'input started'
     end
 
     # Stop everything. Has the effect of draining all the Queues and waiting on
@@ -141,7 +150,7 @@ module Franz
       @watch.stop
       @tail.stop
       @agg.stop
-      log.debug 'stopped input'
+      log.info event: 'input stopped'
       return state
     end
 
@@ -162,14 +171,12 @@ module Franz
     def checkpoint
       old_checkpoints = Dir[@checkpoint_glob].sort_by { |p| File.mtime p }
       path = @checkpoint_path % Time.now
-      begin
-        File.open(path, 'w') { |f| f.write Marshal.dump(state) }
-        old_checkpoints.pop # Keep last two checkpoints
-        old_checkpoints.map { |c| FileUtils.rm c }
-        log.info 'Wrote %s' % path.inspect
-      rescue Errno::EMFILE
-        log.warn 'Could not write checkpoint (too many open files)'
-      end
+      File.open(path, 'w') { |f| f.write Marshal.dump(state) }
+      old_checkpoints.pop # Keep last two checkpoints
+      old_checkpoints.map { |c| FileUtils.rm c }
+      log.info \
+        event: 'input checkpoint saved',
+        checkpoint: path
     end
 
   private

data/lib/franz/output.rb

@@ -34,17 +34,18 @@ module Franz
 
       @logger = opts[:logger]
 
-      rabbit = Bunny.new opts[:output][:connection].merge \
-        automatically_recover: true,
-        threaded: true,
-        heartbeat: 90
+      rabbit = Bunny.new opts[:output][:connection].merge({
+        network_recovery_interval: 10.0,
+        continuation_timeout: 10_000,
+        threaded: false
+      })
 
       rabbit.start
 
       channel  = rabbit.create_channel
       exchange = opts[:output][:exchange].delete(:name)
       exchange = channel.exchange exchange, \
-        opts[:output][:exchange].merge(type: 'x-consistent-hash')
+        { type: 'x-consistent-hash' }.merge(opts[:output][:exchange])
 
       @stop = false
       @foreground = opts[:foreground]
@@ -53,15 +54,38 @@ module Franz
         rand = Random.new
         until @stop
           event = opts[:input].shift
-          event[:tags] = opts[:tags] unless opts[:tags].empty?
-          log.trace 'publishing event=%s' % event.inspect
+
+          event[:path] = event[:path].sub('/home/denimuser/seam-builds/rel', '')
+          event[:path] = event[:path].sub('/home/denimuser/seam-builds/live', '')
+          event[:path] = event[:path].sub('/home/denimuser/seam-builds/beta', '')
+          event[:path] = event[:path].sub('/home/denimuser/builds/rel', '')
+          event[:path] = event[:path].sub('/home/denimuser/builds/live', '')
+          event[:path] = event[:path].sub('/home/denimuser/builds/beta', '')
+          event[:path] = event[:path].sub('/home/denimuser/cobalt-builds/rel', '')
+          event[:path] = event[:path].sub('/home/denimuser/cobalt-builds/live', '')
+          event[:path] = event[:path].sub('/home/denimuser/cobalt-builds/beta', '')
+          event[:path] = event[:path].sub('/home/denimuser/rivet-builds', '')
+          event[:path] = event[:path].sub('/home/denimuser/denim/logs', '')
+          event[:path] = event[:path].sub('/home/denimuser/seam/logs', '')
+          event[:path] = event[:path].sub('/home/denimuser/rivet/bjn/logs', '')
+          event[:path] = event[:path].sub('/home/denimuser', '')
+          event[:path] = event[:path].sub('/var/log', '')
+
+          log.trace \
+            event: 'publish',
+            raw: event
+
           exchange.publish \
             JSON::generate(event),
-            routing_key: rand.rand(1_000_000),
+            routing_key: rand.rand(10_000),
             persistent: false
         end
       end
 
+      log.debug \
+        event: 'output started',
+        foreground: @foreground
+
       @thread.join if @foreground
     end
 
@@ -77,6 +101,7 @@ module Franz
       return if @foreground
       @foreground = true
       @thread.kill
+      log.debug event: 'output stopped'
     end
 
   private

data/lib/franz/sash.rb

@@ -21,6 +21,7 @@ module Franz
       @mutex = Mutex.new
       @mtime = Hash.new { |default, key| default[key] = nil }
       @hash  = Hash.new { |default, key| default[key] = []  }
+      @size  = Hash.new { |default, key| default[key] = 0   }
     end
 
     # Grab a list of known keys.
@@ -37,6 +38,7 @@ module Franz
     def insert key, value
       @mutex.synchronize do
         @hash[key] << value
+        @size[key] += 1
         @mtime[key] = Time.now
       end
       return value
@@ -54,7 +56,10 @@ module Franz
     # @param [Object] key
     #
     # @return [Array<Object>]
-    def remove key ; @hash.delete(key) end
+    def remove key
+      @size[key] -= 1
+      @hash.delete(key)
+    end
 
     # Return the last time the key's value buffer was modified.
     #
@@ -73,9 +78,19 @@ module Franz
       @mutex.synchronize do
         value       = @hash[key]
         @hash[key]  = []
+        @size[key]  = 0
         @mtime[key] = Time.now
       end
       return value
     end
+
+    # Return the size of a key's value buffer.
+    #
+    # @param [Object] key
+    #
+    # @return [Integer]
+    def size key
+      @size[key]
+    end
   end
 end