franz 1.2.7

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 56734aa5910e2ab4f20bca8e21ae5cc76bbcfb9d
+  data.tar.gz: 7bb3ea267a09bdeb68b1e0320daacd04c760890a
+SHA512:
+  metadata.gz: 58ceaffc0dc31e5fc7ea08c77d38c32eb7ace575f000182bb629d0c69888b4e7539e50e83d8a08cbb84e6d15b68b1cd5a03e55ed75f89ed0f35a225cdae79444
+  data.tar.gz: 80c24a9410ea0e4404541f4eecbe2a87becec61fe193758f346a1699d7db145eeba80fde2e79a598df7427421d3ed995a7ad54b77c9a9628bef7146d8a3358a1

data/.gitignore

@@ -0,0 +1,14 @@
+*~
+*.gem
+*.log
+*.out
+*.pid
+*.swp
+*.state
+*.checkpoint
+.DS_Store
+.yardoc
+doc
+pkg
+config.json
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,15 @@
+source 'http://rubygems.org'
+
+gemspec
+
+group :development do
+  gem 'pry'
+  gem 'rake'
+  gem 'yard'
+  gem 'version'
+  gem 'rubygems-tasks'
+end
+
+group :test do
+  gem 'minitest'
+end

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2014 Sean Clemmer and Blue Jeans Network
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.

data/Rakefile

@@ -0,0 +1,52 @@
+require 'rubygems'
+require 'bundler'
+require 'rake'
+
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/test*.rb']
+  test.verbose = true
+end
+
+task :default => :test
+
+
+require 'yard'
+YARD::Rake::YardocTask.new do |t|
+  t.files = %w[ --readme Readme.md lib/**/*.rb - VERSION ]
+end
+
+
+require 'rubygems/tasks'
+Gem::Tasks.new({
+  push: false,
+  sign: {}
+}) do |tasks|
+  tasks.console.command = 'pry'
+end
+Gem::Tasks::Sign::Checksum.new sha2: true
+
+
+require 'rake/version_task'
+Rake::VersionTask.new
+
+
+desc "Upload build artifacts to WOPR"
+task :upload => :build do
+  pkg_name = 'franz-%s.gem' % File.read('VERSION').strip
+  pkg_path = File.join 'pkg', pkg_name
+
+  require 'net/ftp'
+  ftp = Net::FTP.new
+  ftp.connect '10.4.4.15', 8080
+  ftp.login
+  ftp.passive
+  begin
+    ftp.put pkg_path
+    ftp.sendcmd("SITE CHMOD 0664 #{pkg_name}")
+  ensure
+    ftp.close
+  end
+end

data/Readme.md

@@ -0,0 +1,50 @@
+# Franz
+
+Franz ships line-oriented log files to [RabbitMQ](http://www.rabbitmq.com/).
+Think barebones [logstash](http://logstash.net/) in pure Ruby with more modest
+compute and memory requirements.
+
+This is really to avoid the JVM tax, but you probably still want logstash agents
+doing the bulk of the log processing. Using this setup, RabbitMQ and logstash
+may be scaled and restarted independently, so new configurations may be applied
+without interrupting those precious log hosts.
+
+Even so, Franz was designed to be interruped. Before exiting, Franz drains his
+event queues and write any "leftover" state disk. When he's called next, he picks
+up those leftovers and continues as if he were paused.
+
+He's also got a couple of improvements over logstash. Let's discuss!
+
+
+## Improvements
+
+First let me say logstash is an awesome hunk of software thanks to the hard
+work of Jordan Sissel and the entire logstash community. Keep it up!
+
+### Multiline Flush
+
+Anyone familiar with multiline codecs and filters in logstash is familiar with
+the multiline flush issue: You finish writing your log file, you close it and
+wait for it to make it through logstash, but hold up. Where's the last line?
+That's right, stuck. Because logstash is expecting a steady stream of events and
+that last one is being buffered so logstash can decide whether its a multiline
+event. Yup, there's an outstanding issue: [LOGSTASH-271](https://logstash.jira.com/browse/LOGSTASH-271).
+Yup, there's a fix: [Pull #1260](https://github.com/elasticsearch/logstash/pull/1260).
+But it's not yet officially sanctioned. Such is life. At any rate, you don't
+have to deal with this issue in Franz, he flushes inactive buffers after a time.
+Easy-peasy, lemon-squeezy.
+
+### File Handle Eviction
+
+Now I'm not actually sure this issue affects logstash proper, but it's one you
+might face if you decide to write your own, so here goes: If you're tailing a
+bunch of files and you never let go of their file handles, you might very well
+exhaust your ulimit after running for a while. Because Franz is designed to be
+a daemon, he releases or "evicts" file handles after a period of inactivity.
+
+### Sequential Identifiers
+
+Okay one last feature: Every log event is assigned a sequential identifier
+according to its path (and implicitly, host) in the `@seq` field. This is useful
+if you expect your packets to get criss-crossed and you want to reconstruct the
+events in order without relying on timestamps, which you shouldn't.

data/VERSION

@@ -0,0 +1 @@
+1.2.7

data/bin/franz

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+require 'json'
+require 'thread'
+require 'logger'
+
+require 'bunny'
+require 'trollop'
+require 'franz'
+
+
+# If we find a local config, try to use it
+# N.B. We'll fall back to the last guy listed here
+config = nil
+%w[
+  config.json
+  %{HOME}/.franz.json
+  /etc/franz/franz.json
+].each do |path|
+  config = path % ENV rescue next
+  break if File.exist? config
+end
+
+
+# Franz really only accepts a config file as an option, and that config file
+# has got to conform to a certain format. If you're unsure, just look at the
+# default options hashes in both Franz::Input and Franz::Output
+opts = Trollop::options(ARGV) do
+  version Franz::VERSION
+  banner Franz::ART + "\n\n" + <<-EOS.gsub(/^    /, '')
+    #{Franz::SUMMARY}
+
+    Usage: franz [<options>]
+
+    Options:
+  EOS
+  opt :config, 'Configuration file to use', type: :string, default: config
+  opt :debug,  'Enable debugging output', default: false
+  opt :trace,  'Enable trace output', default: false
+  opt :log,    'Log to file, not STDOUT', type: :string, default: nil
+end
+
+Thread.abort_on_exception = true # Die quickly and with great ceremony
+
+config = Franz::Config.new opts[:config]
+logger = Franz::Logger.new opts[:debug], opts[:trace], opts[:log]
+
+io_bound = config[:output][:bound] || 10_000
+
+begin
+  io = SizedQueue.new io_bound
+
+  # Now we'll connect to our output, RabbitMQ. This creates a new thread in the
+  # background, which will consume the events generated by our input on io
+  fout = Franz::Output.new \
+    input: io,
+    output: config[:output][:rabbitmq],
+    logger: logger,
+    tags: config[:output][:tags]
+
+  # Franz has only one kind of input, plain text files.
+  fin = Franz::Input.new \
+    input: config[:input],
+    output: io,
+    logger: logger,
+    checkpoint: config[:checkpoint],
+    checkpoint_interval: config[:checkpoint_interval]
+
+  # Remember, both the input and output were started up in background threads,
+  # so we'll have to wait here in main or else we'll just exit.
+  fout.join
+
+rescue SignalException => e
+  logger.fatal "#{e.inspect} #{$!}\n\t#{$@ * "\n\t"}"
+rescue SystemExit, Interrupt => e
+  logger.fatal "#{e.inspect} #{$!}\n\t#{$@ * "\n\t"}"
+ensure
+  logger.info 'Draining. This may take a while...'
+  fin.stop
+  fin.checkpoint
+  logger.info 'Bye!'
+end

data/franz.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path(File.join('..', 'lib'), __FILE__)
+require 'franz/metadata'
+
+Gem::Specification.new do |s|
+  s.name        = 'franz'
+  s.version     = Franz::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.author      = Franz::AUTHOR
+  s.email       = Franz::EMAIL
+  s.summary     = Franz::SUMMARY
+  s.description = Franz::SUMMARY + '.'
+
+  s.add_runtime_dependency 'bunny', '~> 1'
+  s.add_runtime_dependency 'buftok', '~> 0'
+  s.add_runtime_dependency 'trollop', '~> 2'
+  s.add_runtime_dependency 'colorize', '~> 0'
+  s.add_runtime_dependency 'deep_merge', '~> 1'
+  s.add_runtime_dependency 'consistent-hashing', '~> 1'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- test/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File::basename(f) }
+  s.require_paths = %w[ lib ]
+end

data/lib/franz.rb

@@ -0,0 +1,10 @@
+require_relative 'franz/agg'
+require_relative 'franz/config'
+require_relative 'franz/discover'
+require_relative 'franz/input'
+require_relative 'franz/logger'
+require_relative 'franz/metadata'
+require_relative 'franz/output'
+require_relative 'franz/tail'
+require_relative 'franz/tail_pool'
+require_relative 'franz/watch'

data/lib/franz/agg.rb

@@ -0,0 +1,158 @@
+require 'logger'
+require 'thread'
+require 'socket'
+require 'pathname'
+
+require_relative 'sash'
+
+module Franz
+
+  # Agg mostly aggregates Tail events by applying the multiline filter, but it
+  # also applies the "host" and "type" fields. Basically, it does all the post-
+  # processing after we've retreived a line from a file.
+  class Agg
+    @@host = Socket.gethostname # We'll apply the hostname to all events
+
+    attr_reader :seqs
+
+    # Start a new Agg thread in the background.
+    #
+    # @param [Hash] opts options for the aggregator
+    # @option opts [Array<Hash>] :configs ([]) file input configuration
+    # @option opts [Queue] :tail_events ([]) "input" queue from Tail
+    # @option opts [Queue] :agg_events ([]) "output" queue
+    # @option opts [Integer] :flush_interval (5) seconds between flushes
+    # @option opts [Hash<Path,Fixnum>] :seqs ({}) internal "seqs" state
+    # @option opts [Logger] :logger (Logger.new(STDOUT)) logger to use
+    def initialize opts={}
+      @configs        = opts[:configs]        || Array.new
+      @tail_events    = opts[:tail_events]    || []
+      @agg_events     = opts[:agg_events]     || []
+      @flush_interval = opts[:flush_interval] || 10
+      @seqs           = opts[:seqs]           || Hash.new
+      @logger         = opts[:logger]         || Logger.new(STDOUT)
+
+      @types  = Hash.new
+      @lock   = Mutex.new
+      @buffer = Franz::Sash.new
+      @stop   = false
+
+      @t1 = Thread.new do
+        log.debug 'starting agg-flush'
+        until @stop
+          flush
+          sleep flush_interval
+        end
+        sleep flush_interval
+        flush
+      end
+
+      @t2 = Thread.new do
+        log.debug 'starting agg-capture'
+        until @stop
+          capture
+        end
+      end
+
+      log.debug 'started agg'
+    end
+
+    # Stop the Agg thread. Effectively only once.
+    #
+    # @return [Hash] internal "seqs" state
+    def stop
+      return state if @stop
+      @stop = true
+      @t2.kill
+      @t1.join
+      log.debug 'stopped agg'
+      return state
+    end
+
+    # Return the internal "seqs" state
+    def state
+      return @seqs.dup
+    end
+
+  private
+    attr_reader :configs, :tail_events, :agg_events, :flush_interval, :seqs, :types, :lock, :buffer
+
+    def log ; @logger end
+
+    def type path
+      begin
+        @types.fetch path
+      rescue KeyError
+        configs.each do |config|
+          type = config[:type] if config[:includes].any? { |glob|
+            included = File.fnmatch? glob, path
+            excludes = !config[:excludes].nil?
+            excluded = excludes && config[:excludes].any? { |exlude|
+              File.fnmatch? exlude, File::basename(path)
+            }
+            included && !excluded
+          }
+          return @types[path] = type unless type.nil?
+        end
+        log.error 'Could not identify type for path=%s' % path
+      end
+    end
+
+    def config path
+      configs.select { |c| c[:type] == type(path) }.shift
+    end
+
+    def seq path
+      seqs[path] = seqs.fetch(path, 0) + 1
+    end
+
+    def real_path path
+      Pathname.new(path).realpath.to_s rescue path
+    end
+
+    def enqueue path, message
+      p = real_path path
+      t = type path
+      s = seq path
+      m = message.encode 'UTF-8', invalid: :replace, undef: :replace, replace: '?'
+      log.trace 'enqueue type=%s path=%s seq=%d message=%s' % [
+        t.inspect, p.inspect, s.inspect, m.inspect
+      ]
+      agg_events.push path: p, message: m, type: t, host: @@host, '@seq' => s
+    end
+
+    def capture
+      event = tail_events.shift
+      log.trace 'received path=%s line=%s' % [
+        event[:path], event[:line]
+      ]
+      multiline = config(event[:path])[:multiline]
+      if multiline.nil?
+        enqueue event[:path], event[:line] unless event[:line].empty?
+      else
+        lock.synchronize do
+          if event[:line] =~ multiline
+            buffered = buffer.flush(event[:path])
+            lines    = buffered.map { |e| e[:line] }.join("\n")
+            enqueue event[:path], lines unless lines.empty?
+          end
+          buffer.insert event[:path], event
+        end
+      end
+    end
+
+    def flush
+      lock.synchronize do
+        started = Time.now
+        buffer.keys.each do |path|
+          if started - buffer.mtime(path) >= flush_interval
+            log.trace 'flushing path=%s' % path.inspect
+            buffered = buffer.remove(path)
+            lines    = buffered.map { |e| e[:line] }.join("\n")
+            enqueue path, lines unless lines.empty?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/franz/config.rb

@@ -0,0 +1,24 @@
+require 'json'
+
+module Franz
+
+  # All things configuration.
+  class Config
+
+    # Load a config file path into a Hash, converting to some native types where
+    # appropriate (e.g. a String denoting a Regexp will become Regexp).
+    #
+    # @param path [String] path to a config file
+    #
+    # @return [Hash] config compiled into a native Hash
+    def self.new path
+      config = JSON::parse File.read(path), symbolize_names: true
+      config[:input][:configs].map! do |input|
+        input[:multiline] = Regexp.new input[:multiline] if input.has_key?(:multiline)
+        input[:type] = input[:type].to_sym
+        input
+      end
+      return config
+    end
+  end
+end