franz 1.6.9 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 02a31ad79327096a6556900d4d5e9c98cb10cbf5
-  data.tar.gz: 6f297b8ba0b6167067cc99172712abd140f008c6
+  metadata.gz: 3daab71d5207b3f6b7e32c4799ada5d1c1597f64
+  data.tar.gz: f5440a6bff60093e02925e6e0e81804a97ff730c
 SHA512:
-  metadata.gz: 528eba213321f9c767241e0ffee5473393d6b0ee7c2ac47ef3497b33e9e55037e830c15bc480f9433849a66fac832598a4a298ae5e7a23e607c4d802251a596b
-  data.tar.gz: b36855cd3a21be372e7d65cccec9b6a2cd4cfcf8ff3892baa776ee556f2abd0f647e3d1b05251c3a6f091572270e252cfba6e02b7467e418c971a307af62262d
+  metadata.gz: a8e79de017d808a8ea074a313d7fa80de03946174dcb9533afee6d2480a595f882abd829f5c75b100a5ee0f725c36d9dfd438407f7377ad051035fc18e045fd2
+  data.tar.gz: 5911d098549d5afa4061bb6e7327355bdabc094d096b3c9b1bc9fa7d839b95e20b07bf857ec04f8b645e05a08b384cc89dd705917e0cb51d1298396391a97ca8

data/Readme.md

@@ -9,9 +9,9 @@ doing the bulk of the log processing. Using this setup, RabbitMQ and logstash
 may be scaled and restarted independently, so new configurations may be applied
 without interrupting those precious log hosts.
 
-Even so, Franz was designed to be interruped. Before exiting, Franz drains his
-event queues and write any "leftover" state disk. When he's called next, he picks
-up those leftovers and continues as if he were paused.
+Even so, Franz was designed to be interruped. Before exiting, Franz keeps a log
+of checkpoints, which are used to restore application state in the event of a
+crash.
 
 He's also got a couple of improvements over logstash. Let's discuss!
 
@@ -19,7 +19,7 @@ He's also got a couple of improvements over logstash. Let's discuss!
 ## Improvements
 
 First let me say logstash is an awesome hunk of software thanks to the hard
-work of Jordan Sissel and the entire logstash community. Keep it up!
+work of Jordan Sissel and the entire logstash community.
 
 ### Multiline Flush
 
@@ -141,8 +141,12 @@ It's kinda like a JSON version of the Logstash config language:
         "play_catchup?": true    // Pick up where we left off
       },
 
-      // Only RabbitMQ is supported at the moment
+
+      // If you provide both RabbitMQ and Kafka configurations, Franz will
+      // prefer RabbitMQ. If you provide neither, events are printed to STDOUT
       "output": {
+
+        // RabbitMQ
         "rabbitmq": {
 
           // Must be a consistently-hashed exchange!
@@ -170,6 +174,20 @@ It's kinda like a JSON version of the Logstash config language:
           }
         },
 
+        // Kafka (experimental)
+        "kafka": {
+          "client_id": "hostname",
+          "cluster": [ "localhost:9092" ],
+          "type": "sync",
+          "compression_codec": "snappy"
+          "metadata_refresh_interval_ms": 600000,
+          "max_send_retries": 3,
+          "retry_backoff_ms": 100,
+          "required_acks": 0,
+          "ack_timeout_ms": 1500,
+          "socket_timeout_ms": 10000
+        }
+
         // Advanced configuration (optional)
         "stats_interval": 60, // Emit statistics periodically
         "bound": 25000,       // Limit output queue size
@@ -193,4 +211,14 @@ At Blue Jeans, we deploy Franz with Upstart. Here's a minimal config:
     exec franz
 
 We actually use the [`bjn_franz` cookbook](https://github.com/sczizzo/bjn-franz-cookbook)
-for Chef.
+for Chef.
+
+### Changelog
+
+#### v2.0.0
+
+- Added new outputs: `StdOut`, `Kafka` (experimental)
+
+#### v1
+
+Intial implementation of the file-to-RabbitMQ pipeline

data/VERSION

@@ -1 +1 @@
-1.6.9
+2.0.0

data/bin/franz

@@ -69,14 +69,33 @@ statz = Franz::Stats.new \
   interval: (config[:output][:stats_interval] || 300),
   logger: logger
 
-# Now we'll connect to our output, RabbitMQ. This creates a new thread in the
-# background, which will consume the events generated by our input on io
-Franz::Output.new \
-  input: io,
-  output: config[:output][:rabbitmq],
-  logger: logger,
-  tags: config[:output][:tags],
-  statz: statz
+
+# Now we'll connect to our output. This creates a new thread in the background,
+# which will consume the events generated by our input on io
+if config[:output][:rabbitmq]
+  Franz::Output::RabbitMQ.new \
+    input: io,
+    output: config[:output][:rabbitmq],
+    logger: logger,
+    tags: config[:output][:tags],
+    statz: statz
+
+elsif config[:output][:kafka]
+  Franz::Output::Kafka.new \
+    input: io,
+    output: config[:output][:kafka],
+    logger: logger,
+    tags: config[:output][:tags],
+    statz: statz
+
+else
+  Franz::Output::StdOut.new \
+    input: io,
+    logger: logger,
+    tags: config[:output][:tags],
+    statz: statz
+end
+
 
 # Franz has only one kind of input, plain text files.
 Franz::Input.new \
@@ -87,6 +106,8 @@ Franz::Input.new \
   checkpoint_interval: config[:checkpoint_interval],
   statz: statz
 
+
+
 # Ensure memory doesn't grow too large (> 1GB by default)
 def mem_kb ; `ps -o rss= -p #{$$}`.strip.to_i ; end
 

data/franz.gemspec

@@ -19,6 +19,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'colorize', '~> 0.7.0'
   s.add_runtime_dependency 'deep_merge', '~> 1.0.0'
   s.add_runtime_dependency 'eventmachine', '= 1.0.5'
+  s.add_runtime_dependency 'poseidon', '~> 0.0.5'
+  s.add_runtime_dependency 'snappy', '~> 0.0.11'
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- test/*`.split("\n")

data/lib/franz/output/kafka.rb

@@ -0,0 +1,135 @@
+require 'thread'
+require 'json'
+
+require 'poseidon'
+require 'deep_merge'
+
+
+module Franz
+  module Output
+
+    # Kafka output for Franz.
+    class Kafka
+      @@host = Socket.gethostname # We'll apply the hostname to all events
+
+
+      # Start a new output in the background. We'll consume from the input queue
+      # and ship events to STDOUT.
+      #
+      # @param [Hash] opts options for the output
+      # @option opts [Queue] :input ([]) "input" queue
+      # @option opts [Queue] :output ([]) "output" configuration
+      def initialize opts={}
+        opts = {
+          logger: Logger.new(STDOUT),
+          tags: [],
+          input: [],
+          output: {
+            flush_interval: 10,
+            flush_size: 500,
+            client_id: @@host,
+            cluster: %w[ localhost:9092 ],
+            type: 'sync',
+            compression_codec: 'snappy',
+            metadata_refresh_interval_ms: 600000,
+            max_send_retries: 3,
+            retry_backoff_ms: 100,
+            required_acks: 0,
+            ack_timeout_ms: 1500,
+            socket_timeout_ms: 10000
+          }
+        }.deep_merge!(opts)
+
+        @statz = opts[:statz] || Franz::Stats.new
+        @statz.create :num_output, 0
+
+        @logger = opts[:logger]
+
+        @stop = false
+        @foreground = opts[:foreground]
+
+        @flush_size = opts[:output].delete :flush_size
+        @flush_interval = opts[:output].delete :flush_interval
+
+        kafka_cluster = opts[:output].delete :cluster
+        kafka_client_id = opts[:output].delete :client_id
+        kafka_config = opts[:output].map { |k,v| v.is_a?(String) ? v.to_sym : v }
+
+        @kafka = Poseidon::Producer.new \
+          kafka_cluster,
+          kafka_client_id,
+          Hash[kafka_config]
+
+        @lock = Mutex.new
+        @messages = []
+
+
+        @thread = Thread.new do
+          loop do
+            ready_messages = []
+            @lock.synchronize do
+              ready_messages = @messages
+              @messages = []
+            end
+            @kafka.send_messages ready_messages unless ready_messages.empty?
+            log.debug \
+              event: 'periodic flush',
+              num_messages: ready_messages.size
+            sleep @flush_interval
+          end
+        end
+
+
+        @thread = Thread.new do
+          until @stop
+            event = opts[:input].shift
+
+            log.trace \
+              event: 'publish',
+              raw: event
+
+            payload = JSON::generate(event)
+            @messages << Poseidon::MessageToSend.new(event[:type].to_s, payload)
+
+            @statz.inc :num_output
+
+            if @statz.get(:num_output) % @flush_size == 0
+              @kafka.send_messages @messages unless @messages.empty?
+              log.debug \
+                event: 'flush',
+                num_messages: @messages.size
+              @messages = []
+            end
+
+          end
+        end
+
+        log.info event: 'output started'
+
+        @thread.join if @foreground
+      end
+
+
+      # Join the Output thread. Effectively only once.
+      def join
+        return if @foreground
+        @foreground = true
+        @thread.join
+      end
+
+
+      # Stop the Output thread. Effectively only once.
+      def stop
+        return if @foreground
+        @foreground = true
+        @thread.kill
+        log.info event: 'output stopped'
+      end
+
+
+    private
+      def log ; @logger end
+
+    end
+  end
+end

data/lib/franz/output/rabbitmq.rb

@@ -0,0 +1,101 @@
+require 'json'
+
+require 'bunny'
+require 'deep_merge'
+
+
+module Franz
+  module Output
+
+    # RabbitMQ output for Franz. You must declare an x-consistent-hash type
+    # exchange, as we generate random Integers for routing keys.
+    class RabbitMQ
+
+      # Start a new output in the background. We'll consume from the input queue
+      # and ship events to the configured RabbitMQ cluster.
+      #
+      # @param [Hash] opts options for the output
+      # @option opts [Queue] :input ([]) "input" queue
+      # @option opts [Hash] :output ({}) "output" configuration
+      def initialize opts={}
+        opts = {
+          logger: Logger.new(STDOUT),
+          tags: [],
+          input: [],
+          output: {
+            exchange: {
+              name: 'test',
+              durable: true
+            },
+            connection: {
+              host: 'localhost',
+              port: 5672
+            }
+          }
+        }.deep_merge!(opts)
+
+        @statz = opts[:statz] || Franz::Stats.new
+        @statz.create :num_output, 0
+
+        @logger = opts[:logger]
+
+        rabbit = Bunny.new opts[:output][:connection].merge({
+          network_recovery_interval: 10.0,
+          continuation_timeout: 10_000,
+          threaded: false,
+          logger: @logger
+        })
+
+        rabbit.start
+
+        channel  = rabbit.create_channel
+        exchange = opts[:output][:exchange].delete(:name)
+        exchange = channel.exchange exchange, \
+          { type: 'x-consistent-hash' }.merge(opts[:output][:exchange])
+
+        @stop = false
+        @foreground = opts[:foreground]
+
+        @thread = Thread.new do
+          rand = Random.new
+          until @stop
+            event = opts[:input].shift
+
+            log.trace \
+              event: 'publish',
+              raw: event
+
+            exchange.publish \
+              JSON::generate(event),
+              routing_key: rand.rand(10_000),
+              persistent: false
+
+            @statz.inc :num_output
+          end
+        end
+
+        log.info event: 'output started'
+
+        @thread.join if @foreground
+      end
+
+      # Join the Output thread. Effectively only once.
+      def join
+        return if @foreground
+        @foreground = true
+        @thread.join
+      end
+
+      # Stop the Output thread. Effectively only once.
+      def stop
+        return if @foreground
+        @foreground = true
+        @thread.kill
+        log.info event: 'output stopped'
+      end
+
+    private
+      def log ; @logger end
+    end
+  end
+end

data/lib/franz/output/stdout.rb

@@ -0,0 +1,70 @@
+require 'json'
+
+require 'deep_merge'
+
+
+module Franz
+  module Output
+
+    # STDOUT output for Franz.
+    class StdOut
+
+      # Start a new output in the background. We'll consume from the input queue
+      # and ship events to STDOUT.
+      #
+      # @param [Hash] opts options for the output
+      # @option opts [Queue] :input ([]) "input" queue
+      def initialize opts={}
+        opts = {
+          logger: Logger.new(STDOUT),
+          tags: [],
+          input: []
+        }.deep_merge!(opts)
+
+        @statz = opts[:statz] || Franz::Stats.new
+        @statz.create :num_output, 0
+
+        @logger = opts[:logger]
+
+        @stop = false
+        @foreground = opts[:foreground]
+
+        @thread = Thread.new do
+          until @stop
+            event = opts[:input].shift
+
+            log.trace \
+              event: 'publish',
+              raw: event
+
+            puts JSON::generate(event)
+
+            @statz.inc :num_output
+          end
+        end
+
+        log.info event: 'output started'
+
+        @thread.join if @foreground
+      end
+
+      # Join the Output thread. Effectively only once.
+      def join
+        return if @foreground
+        @foreground = true
+        @thread.join
+      end
+
+      # Stop the Output thread. Effectively only once.
+      def stop
+        return if @foreground
+        @foreground = true
+        @thread.kill
+        log.info event: 'output stopped'
+      end
+
+    private
+      def log ; @logger end
+    end
+  end
+end

data/lib/franz/output.rb

@@ -1,121 +1,4 @@
-require 'json'
-
-require 'bunny'
-require 'deep_merge'
-
 require_relative 'stats'
-
-
-module Franz
-
-  # RabbitMQ output for Franz. You must declare an x-consistent-hash type
-  # exchange, as we generate random Integers for routing keys.
-  class Output
-
-    # Start a new output in the background. We'll consume from the input queue
-    # and ship events to the configured RabbitMQ cluster.
-    #
-    # @param [Hash] opts options for the output
-    # @option opts [Queue] :input ([]) "input" queue
-    # @option opts [Hash] :output ({}) "output" configuration
-    def initialize opts={}
-      opts = {
-        logger: Logger.new(STDOUT),
-        tags: [],
-        input: [],
-        output: {
-          exchange: {
-            name: 'test',
-            durable: true
-          },
-          connection: {
-            host: 'localhost',
-            port: 5672
-          }
-        }
-      }.deep_merge!(opts)
-
-      @statz = opts[:statz] || Franz::Stats.new
-      @statz.create :num_output, 0
-
-      @logger = opts[:logger]
-
-      rabbit = Bunny.new opts[:output][:connection].merge({
-        network_recovery_interval: 10.0,
-        continuation_timeout: 10_000,
-        threaded: false,
-        logger: @logger
-      })
-
-      rabbit.start
-
-      channel  = rabbit.create_channel
-      exchange = opts[:output][:exchange].delete(:name)
-      exchange = channel.exchange exchange, \
-        { type: 'x-consistent-hash' }.merge(opts[:output][:exchange])
-
-      @stop = false
-      @foreground = opts[:foreground]
-
-      @thread = Thread.new do
-        rand = Random.new
-        until @stop
-          event = opts[:input].shift
-
-          if event[:path]
-            # Can't use sub!, because :path is frozen
-            event[:path] = event[:path].sub '/home/denimuser/seam-builds/rel', ''
-            event[:path] = event[:path].sub '/home/denimuser/seam-builds/live', ''
-            event[:path] = event[:path].sub '/home/denimuser/seam-builds/beta', ''
-            event[:path] = event[:path].sub '/home/denimuser/builds/rel', ''
-            event[:path] = event[:path].sub '/home/denimuser/builds/live', ''
-            event[:path] = event[:path].sub '/home/denimuser/builds/beta', ''
-            event[:path] = event[:path].sub '/home/denimuser/cobalt-builds/rel', ''
-            event[:path] = event[:path].sub '/home/denimuser/cobalt-builds/live', ''
-            event[:path] = event[:path].sub '/home/denimuser/cobalt-builds/beta', ''
-            event[:path] = event[:path].sub '/home/denimuser/rivet-builds', ''
-            event[:path] = event[:path].sub '/home/denimuser/denim/logs', ''
-            event[:path] = event[:path].sub '/home/denimuser/seam/logs', ''
-            event[:path] = event[:path].sub '/home/denimuser/rivet/bjn/logs', ''
-            event[:path] = event[:path].sub '/home/denimuser', ''
-            event[:path] = event[:path].sub '/var/log/franz', ''
-            event[:path] = event[:path].sub '/var/log', ''
-          end
-
-          log.trace \
-            event: 'publish',
-            raw: event
-
-          exchange.publish \
-            JSON::generate(event),
-            routing_key: rand.rand(10_000),
-            persistent: false
-
-          @statz.inc :num_output
-        end
-      end
-
-      log.info event: 'output started'
-
-      @thread.join if @foreground
-    end
-
-    # Join the Output thread. Effectively only once.
-    def join
-      return if @foreground
-      @foreground = true
-      @thread.join
-    end
-
-    # Stop the Output thread. Effectively only once.
-    def stop
-      return if @foreground
-      @foreground = true
-      @thread.kill
-      log.info event: 'output stopped'
-    end
-
-  private
-    def log ; @logger end
-  end
-end
+require_relative 'output/kafka'
+require_relative 'output/stdout'
+require_relative 'output/rabbitmq'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: franz
 version: !ruby/object:Gem::Version
-  version: 1.6.9
+  version: 2.0.0
 platform: ruby
 authors:
 - Sean Clemmer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-09 00:00:00.000000000 Z
+date: 2015-03-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slog
@@ -94,6 +94,34 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.0.5
+- !ruby/object:Gem::Dependency
+  name: poseidon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+- !ruby/object:Gem::Dependency
+  name: snappy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.11
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.11
 description: Aggregate log file events and send them elsewhere.
 email: sclemmer@bluejeans.com
 executables:
@@ -118,6 +146,9 @@ files:
 - lib/franz/input_config.rb
 - lib/franz/metadata.rb
 - lib/franz/output.rb
+- lib/franz/output/kafka.rb
+- lib/franz/output/rabbitmq.rb
+- lib/franz/output/stdout.rb
 - lib/franz/sash.rb
 - lib/franz/stats.rb
 - lib/franz/tail.rb