franz 1.2.7

data/lib/franz/discover.rb

@@ -0,0 +1,108 @@
+require 'logger'
+
+
+# Discover performs half of file existence detection by expanding globs and
+# keeping track of files known to Franz. Discover requires a deletions Queue to
+# maintain this state, so it's fairly useless without a Watch.
+class Franz::Discover
+
+  # Start a new Discover thread in the background.
+  #
+  # @param [Hash] opts options for the discovery
+  # @option opts [Array<Hash>] :configs ([]) file input configuration
+  # @option opts [Queue] :discoveries ([]) "output" queue of discovered paths
+  # @option opts [Queue] :deletions ([]) "input" queue of deleted paths
+  # @option opts [Integer] :discover_interval (5) seconds between discover rounds
+  # @option opts [Array<Path>] :known ([]) internal "known" state
+  # @option opts [Logger] :logger (Logger.new(STDOUT)) logger to use
+  def initialize opts={}
+    @configs           = opts[:configs]           || []
+    @discoveries       = opts[:discoveries]       || []
+    @deletions         = opts[:deletions]         || []
+    @discover_interval = opts[:discover_interval] || 30
+    @ignore_before     = opts[:ignore_before]     || 0
+    @known             = opts[:known]             || []
+    @logger            = opts[:logger]            || Logger.new(STDOUT)
+
+    @configs = configs.map do |config|
+      config[:includes] ||= []
+      config[:excludes] ||= []
+      config
+    end
+
+    @stop = false
+
+    @thread = Thread.new do
+      log.debug 'starting dicover-delete'
+      until @stop
+        until deletions.empty?
+          d = deletions.pop
+          @known.delete d
+          log.debug 'deleted: %s' % d.inspect
+        end
+        discover.each do |discovery|
+          discoveries.push discovery
+          @known.push discovery
+          log.debug 'discovered: %s' % discovery.inspect
+        end
+        sleep discover_interval
+      end
+    end
+
+    log.debug 'started discover'
+  end
+
+  # Stop the Discover thread. Effectively only once.
+  #
+  # @return [Array] internal "known" state
+  def stop
+    return state if @stop
+    @stop = true
+    @thread.join
+    log.debug 'stopped discover'
+    return state
+  end
+
+  # Return the internal "known" state
+  def state
+    return @known.dup
+  end
+
+private
+  attr_reader :configs, :discoveries, :deletions, :discover_interval, :known
+
+  def log ; @logger end
+
+  def discover
+    discovered = []
+    configs.each do |config|
+      config[:includes].each do |glob|
+        expand(glob).each do |path|
+          next if config[:excludes].any? { |exclude|
+            File.fnmatch? exclude, File::basename(path)
+          }
+          next if known.include? path
+          next unless File.file? path
+          next if File.mtime(path).to_i <= @ignore_before
+          discovered.push path
+        end
+      end
+    end
+    return discovered
+  end
+
+  def expand glob
+    dir_glob = File.dirname(glob)
+    file_glob = File.basename(glob)
+    files = []
+    Dir.glob(dir_glob).each do |dir|
+      next unless File::directory?(dir)
+      Dir.foreach(dir) do |fname|
+        next if fname == '.' || fname == '..'
+        next unless File.fnmatch?(file_glob, fname)
+        files << File.join(dir, fname)
+      end
+    end
+    files
+  end
+end

data/lib/franz/input.rb

@@ -0,0 +1,174 @@
+require 'logger'
+require 'fileutils'
+
+require 'deep_merge'
+
+require_relative 'agg'
+require_relative 'tail'
+require_relative 'tail_pool'
+require_relative 'watch'
+require_relative 'discover'
+
+module Franz
+
+  # File input for Franz. Really, the only input for Franz, so I hope you like it.
+  class Input
+    # Start a new input in the background. We'll generate a stream of events by
+    # watching the filesystem for changes (Franz::Discover and Franz::Watch),
+    # tailing files (Franz::Tail), and generating events (Franz::Agg)
+    #
+    # @param [Hash] opts options for the aggregator
+    # @option opts [Hash] :input ({}) "input" configuration
+    # @option opts [Queue] :output ([]) "output" queue
+    # @option opts [Path] :checkpoint ({}) path to checkpoint file
+    # @option opts [Integer] :checkpoint_interval ({}) seconds between checkpoints
+    # @option opts [Logger] :logger (Logger.new(STDOUT)) logger to use
+    def initialize opts={}
+      opts = {
+        checkpoint: 'franz.*.checkpoint',
+        checkpoint_interval: 30,
+        logger: Logger.new(STDOUT),
+        output: [],
+        input: {
+          ignore_before: 0,
+          tail_pool_size: 10,
+          discover_bound: 10_000,
+          watch_bound: 1_000,
+          tail_bound: 1_000,
+          discover_interval: nil,
+          watch_interval: nil,
+          eviction_interval: nil,
+          flush_interval: nil,
+          configs: []
+        }
+      }.deep_merge!(opts)
+
+      @logger = opts[:logger]
+
+      @checkpoint_interval = opts[:checkpoint_interval]
+      @checkpoint_path     = opts[:checkpoint].sub('*', '%d')
+      @checkpoint_glob     = opts[:checkpoint]
+
+      # The checkpoint contains a Marshalled Hash with a compact representation of
+      # stateful inputs to various Franz streaming classes (e.g. the "known" option
+      # to Franz::Discover). This state file is generated automatically every time
+      # the input exits (see below) and also at regular intervals.
+      checkpoints = Dir[@checkpoint_glob].sort_by { |path| File.mtime path }
+      checkpoints = checkpoints.reject { |path| File.zero? path }
+      last_checkpoint_path = checkpoints.pop
+      state = nil
+      unless last_checkpoint_path.nil?
+        last_checkpoint = File.read(last_checkpoint_path)
+        state = Marshal.load last_checkpoint
+        log.debug 'Loaded %s' % last_checkpoint_path.inspect
+      end
+
+      state = state || {}
+      known = state.keys
+      stats, cursors, seqs = {}, {}, {}
+      known.each do |path|
+        cursor        = state[path].delete :cursor
+        seq           = state[path].delete :seq
+        cursors[path] = cursor unless cursor.nil?
+        seqs[path]    = seq    unless seq.nil?
+        stats[path]   = state[path]
+      end
+
+      log.debug 'starting input...'
+
+      discoveries  = SizedQueue.new opts[:input][:discover_bound]
+      deletions    = SizedQueue.new opts[:input][:discover_bound]
+      watch_events = SizedQueue.new opts[:input][:watch_bound]
+      tail_events  = SizedQueue.new opts[:input][:tail_bound]
+
+      log.debug 'starting discover...'
+      @disover = Franz::Discover.new \
+        discoveries: discoveries,
+        deletions: deletions,
+        configs: opts[:input][:configs],
+        discover_interval: opts[:input][:discover_interval],
+        ignore_before: opts[:input][:ignore_before],
+        logger: opts[:logger],
+        known: known
+
+      log.debug 'starting tail...'
+      @tail = Franz::Tail.new \
+        watch_events: watch_events,
+        tail_events: tail_events,
+        eviction_interval: opts[:input][:eviction_interval],
+        logger: opts[:logger],
+        cursors: cursors
+
+      log.debug 'starting agg...'
+      @agg = Franz::Agg.new \
+        configs: opts[:input][:configs],
+        tail_events: tail_events,
+        agg_events: opts[:output],
+        flush_interval: opts[:input][:flush_interval],
+        logger: opts[:logger],
+        seqs: seqs
+
+      log.debug 'starting watch...'
+      @watch = Franz::Watch.new \
+        discoveries: discoveries,
+        deletions: deletions,
+        watch_events: watch_events,
+        watch_interval: opts[:input][:watch_interval],
+        logger: opts[:logger],
+        stats: stats
+
+      @stop = false
+      @t = Thread.new do
+        log.debug 'starting checkpoint'
+        until @stop
+          checkpoint
+          sleep @checkpoint_interval
+        end
+      end
+
+      log.debug 'started input'
+    end
+
+    # Stop everything. Has the effect of draining all the Queues and waiting on
+    # auxilliarly threads (e.g. eviction) to complete full intervals, so it may
+    # ordinarily take tens of seconds, depending on your configuration.
+    #
+    # @return [Hash] compact internal state
+    def stop
+      return state if @stop
+      @stop = true
+      @t.join
+      @watch.stop
+      @tail.stop
+      @agg.stop
+      log.debug 'stopped input'
+      return state
+    end
+
+    # Return a compact representation of internal state
+    def state
+      stats   = @watch.state
+      cursors = @tail.state
+      seqs    = @agg.state
+      stats.keys.each do |path|
+        stats[path] ||= {}
+        stats[path][:cursor] = cursors.fetch(path, nil)
+        stats[path][:seq] = seqs.fetch(path, nil)
+      end
+      return stats
+    end
+
+    # Write a checkpoint file given the current state
+    def checkpoint
+      old_checkpoints = Dir[@checkpoint_glob].sort_by { |p| File.mtime p }
+      path = @checkpoint_path % Time.now
+      File.open(path, 'w') { |f| f.write Marshal.dump(state) }
+      old_checkpoints.pop # Keep last two checkpoints
+      old_checkpoints.map { |c| FileUtils.rm c }
+      log.info 'Wrote %s' % path.inspect
+    end
+
+  private
+    def log ; @logger end
+  end
+end

data/lib/franz/logger.rb

@@ -0,0 +1,66 @@
+require 'logger'
+require 'time'
+
+require 'colorize'
+
+module Franz
+
+  # Extending the Logger with TRACE capabilities
+  class ::Logger
+    SEV_LABEL << 'TRACE'
+    TRACE = SEV_LABEL.index('TRACE') # N.B. TRACE is above other levels
+
+    # Send a TRACE-level log line
+    def trace progname, &block
+      add TRACE, nil, progname, &block if @trace
+    end
+  end
+
+  # A powerful, colorful logger for Franz.
+  class Logger < Logger
+    # Maps each log level to a unique combination of fore- and background colors
+    SEVERITY_COLORS = {
+      'DEBUG' => [ :blue,    :default ],
+      'INFO'  => [ :green,   :default ],
+      'WARN'  => [ :yellow,  :default ],
+      'ERROR' => [ :red,     :default ],
+      'FATAL' => [ :red,     :black   ],
+      'TRACE' => [ :magenta, :default ]
+    }
+
+    # Create a new, colorful logger.
+    #
+    # @param debug [Boolean] enable DEBUG level logs
+    # @param out [File] output destination for logs
+    def initialize debug=false, trace=false, out=nil
+      out ||= $stdout
+      super out
+      colorize
+      @trace = true if trace
+      self.level = ::Logger::INFO
+      self.level = ::Logger::DEBUG if debug
+    end
+
+  private
+    def colorize
+      self.formatter = proc do |severity, datetime, _, message|
+        if level == 1
+          message.to_s.colorize(
+            color: SEVERITY_COLORS[severity.to_s][0],
+            background: SEVERITY_COLORS[severity.to_s][1]
+          ) + "\n"
+        else
+          "%s [%s] %s -- %s\n".colorize(
+            color: SEVERITY_COLORS[severity.to_s][0],
+            background: SEVERITY_COLORS[severity.to_s][1]
+          ) % [
+            severity,
+            datetime.iso8601(6),
+            File::basename(caller[4]),
+            message
+          ]
+        end
+      end
+    end
+  end
+end

data/lib/franz/metadata.rb

@@ -0,0 +1,33 @@
+# Franz ships line-oriented log files to RabbitMQ. Think barebones logstash in
+# pure Ruby with more modest compute and memory requirements.
+module Franz
+
+  # We use a VERSION file to tie into our build pipeline
+  VERSION  = File.read(File.join(File.dirname(__FILE__), '..', '..', 'VERSION')).strip
+
+  # We don't really do all that much, be humble
+  SUMMARY  = 'Aggregate log file events and send them elsewhere'
+
+  # Your benevolent dictator for life
+  AUTHOR   = 'Sean Clemmer'
+
+  # Turn here to strangle your dictator
+  EMAIL    = 'sclemmer@bluejeans.com'
+
+  # Every project deserves its own ASCII art
+  ART      = <<-'EOART' % VERSION
+
+      .--.,
+    ,--.'  \  __  ,-.                 ,---,        ,----,
+    |  | /\/,' ,'/ /|             ,-+-. /  |     .'   .`|
+    :  : :  '  | |' | ,--.--.    ,--.'|'   |  .'   .'  .'
+    :  | |-,|  |   ,'/       \  |   |  ,"' |,---, '   ./
+    |  : :/|'  :  / .--.  .-. | |   | /  | |;   | .'  /
+    |  |  .'|  | '   \__\/: . . |   | |  | |`---' /  ;--,
+    '  : '  ;  : |   ," .--.; | |   | |  |/   /  /  / .`|
+    |  | |  |  , ;  /  /  ,.  | |   | |--'  ./__;     .'
+    |  : \   ---'  ;  :   .'   \|   |/      ;   |  .'
+    |  |,'         |  ,     .-./'---'       `---'
+    `--'            `--`---'                       v%s
+  EOART
+end

data/lib/franz/output.rb

@@ -0,0 +1,81 @@
+require 'json'
+
+require 'bunny'
+require 'deep_merge'
+
+module Franz
+
+  # RabbitMQ output for Franz. You must declare an x-consistent-hash type
+  # exchange, as we generate random Integers for routing keys.
+  class Output
+
+    # Start a new output in the background. We'll consume from the input queue
+    # and ship events to the configured RabbitMQ cluster.
+    #
+    # @param [Hash] opts options for the output
+    # @option opts [Queue] :input ([]) "input" queue
+    # @option opts [Hash] :output ({}) "output" configuration
+    def initialize opts={}
+      opts = {
+        logger: Logger.new(STDOUT),
+        tags: [],
+        input: [],
+        output: {
+          exchange: {
+            name: 'test',
+            durable: true
+          },
+          connection: {
+            host: 'localhost',
+            port: 5672
+          }
+        }
+      }.deep_merge!(opts)
+
+      @logger = opts[:logger]
+
+      rabbit = Bunny.new opts[:output][:connection]
+      rabbit.start
+
+      channel  = rabbit.create_channel
+      exchange = opts[:output][:exchange].delete(:name)
+      exchange = channel.exchange exchange, \
+        opts[:output][:exchange].merge(type: 'x-consistent-hash')
+
+      @stop = false
+      @foreground = opts[:foreground]
+
+      @thread = Thread.new do
+        rand = Random.new
+        until @stop
+          event = opts[:input].shift
+          event[:tags] = opts[:tags] unless opts[:tags].empty?
+          log.trace 'publishing event=%s' % event.inspect
+          exchange.publish \
+            JSON::generate(event),
+            routing_key: rand.rand(1_000_000),
+            persistent: false
+        end
+      end
+
+      @thread.join if @foreground
+    end
+
+    # Join the Output thread. Effectively only once.
+    def join
+      return if @foreground
+      @foreground = true
+      @thread.join
+    end
+
+    # Stop the Output thread. Effectively only once.
+    def stop
+      return if @foreground
+      @foreground = true
+      @thread.kill
+    end
+
+  private
+    def log ; @logger end
+  end
+end