franz 2.1.5 → 2.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1a45e8b1c89419f9b75b0b04fde00fddedb4ac0a
-  data.tar.gz: 011ccd003599089f514a93d0124224c22911d38d
+  metadata.gz: 192d0de743aeb1560eb55145d11b06c2f1f5ba61
+  data.tar.gz: 145887b6c3313ad29f4eb88f9d17df62899a27c2
 SHA512:
-  metadata.gz: c145e7fd8ea9acd1b623bf6ce742bc72ad6adc992c226275c11881f6731b3e2b01bc64fa1e991dfe8e6ff652a074ca261c3900c3a94f2b0301cc7c87ea892bdd
-  data.tar.gz: a8f94d4ac45dbdffc8e920dc899fa23f8cd9e5941a37c7729caaf53dcf78e7f29034f30ba8b4199a05a506315fa41873f4d28eee6cf624d3f818a6533ab7db75
+  metadata.gz: b2a4cde46ea536888c90c4f2fb2611a7f4526840c2a422d85f99b646385b580e48f1eee92fc91c0422c1287d075d8e92530c80f060e0bdeb65cbace48276517f
+  data.tar.gz: fdbb262b5620837f14d92986e12d946decd4b335665953ff7784151dcf6fafee8ba0843cbe14179d9a9e141cdbb2fdabe8ce187043ebc02db3d0ef0ac21787b0

data/Readme.md

@@ -188,8 +188,15 @@ It's kinda like a JSON version of the Logstash config language:
           "socket_timeout_ms": 10000
         },
 
-        // StdOut (implied if neither Kafka nor RabbitMQ configured)
-        "stdout": {},
+        // HTTP (experimental)
+        "http": {
+          "server": "http://localhost:3000",
+          "flush_interval": 5,
+          "flush_size": 500
+        },
+
+        // Device (STDOUT implied if neither Kafka nor RabbitMQ configured)
+        "device": "/dev/stdout",
 
         // Advanced configuration (optional)
         "stats_interval": 60, // Emit statistics periodically
@@ -223,8 +230,10 @@ for Chef.
 - Self-contained packages for OS X and Linux thanks to [Traveling Ruby](http://phusion.github.io/traveling-ruby) and [FPM](https://github.com/jordansissel/fpm)
 - Dockerization using self-contained Linux package [available on the Docker Hub](https://registry.hub.docker.com/u/sczizzo/franz)
 - Running Franz without arguments now prints help text (i.e. `--config` is now required)
+- Rename `StdOut` to `Device`, no longer experimental
 - Allow single state file if no glob is used
 - Handle `EACCESS` on `IO.read` gracefully
+- New `HTTP` output (experimental)
 - Remove default line limit
 
 #### v2.0

data/VERSION

@@ -1 +1 @@
-2.1.5
+2.1.6

data/bin/franz

@@ -41,10 +41,11 @@ config = Franz::Config.new opts[:config]
 level = :info
 level = :debug if opts[:debug]
 level = :trace if opts[:trace]
+device = opts[:log] || $stdout
 colorize, prettify = false, false
-colorize, prettify = true, true if opts[:log].nil?
+colorize, prettify = true, true if device.tty? rescue false
 logger = Slog.new (config[:slog] || {}).merge \
-  out: opts[:log] || $stdout,
+  out: device,
   colorize: colorize,
   prettify: prettify,
   level: level
@@ -81,10 +82,19 @@ elsif config[:output][:kafka]
     tags: config[:output][:tags],
     statz: statz
 
+elsif config[:output][:http]
+  Franz::Output::HTTP.new \
+    input: io,
+    output: config[:output][:http],
+    logger: logger,
+    tags: config[:output][:tags],
+    statz: statz
+
 else
-  Franz::Output::StdOut.new \
+  Franz::Output::Device.new \
     input: io,
     logger: logger,
+    output: config[:output][:device],
     tags: config[:output][:tags],
     statz: statz
 end

data/lib/franz/output.rb

@@ -1,4 +1,5 @@
 require_relative 'stats'
+require_relative 'output/http'
 require_relative 'output/kafka'
-require_relative 'output/stdout'
+require_relative 'output/device'
 require_relative 'output/rabbitmq'

data/lib/franz/output/{stdout.rb → device.rb}

@@ -7,7 +7,7 @@ module Franz
   module Output
 
     # STDOUT output for Franz.
-    class StdOut
+    class Device
 
       # Start a new output in the background. We'll consume from the input queue
       # and ship events to STDOUT.
@@ -18,12 +18,14 @@ module Franz
         opts = {
           logger: Logger.new(STDOUT),
           tags: [],
-          input: []
+          input: [],
+          output: '/dev/stdout'
         }.deep_merge!(opts)
 
         @statz = opts[:statz] || Franz::Stats.new
         @statz.create :num_output, 0
 
+        @device = File.open(opts[:output], 'w')
         @logger = opts[:logger]
 
         @stop = false
@@ -37,7 +39,7 @@ module Franz
               event: 'publish',
               raw: event
 
-            puts JSON::generate(event)
+            @device.puts JSON::generate(event)
 
             @statz.inc :num_output
           end
@@ -60,6 +62,7 @@ module Franz
         return if @foreground
         @foreground = true
         @thread.kill
+        @device.close
         log.info event: 'output stopped'
       end
 

data/lib/franz/output/http.rb

@@ -0,0 +1,128 @@
+require 'net/http'
+require 'thread'
+require 'fiber'
+require 'json'
+
+require 'deep_merge'
+
+
+module Franz
+  module Output
+
+    # HTTP output for Franz.
+    class HTTP
+      # Start a new output in the background. We'll consume from the input queue
+      # and ship events via HTTP.
+      #
+      # @param [Hash] opts options for the output
+      # @option opts [Queue] :input ([]) "input" queue
+      # @option opts [Queue] :output ([]) "output" configuration
+      def initialize opts={}
+        opts = {
+          logger: Logger.new(STDOUT),
+          tags: [],
+          input: [],
+          output: {
+            server: 'http://localhost:3000',
+            flush_size: 500,
+            flush_interval: 10
+          }
+        }.deep_merge!(opts)
+
+        @statz = opts[:statz] || Franz::Stats.new
+        @statz.create :num_output, 0
+
+        @logger = opts[:logger]
+
+        @stop = false
+        @foreground = opts[:foreground]
+
+        server = opts[:output].delete :server
+        @uri   = URI(server)
+        open_uri
+
+        @flush_size = opts[:output][:flush_size]
+        @flush_interval = opts[:output][:flush_interval]
+        @lock = Mutex.new
+        @messages = []
+
+        Thread.new do
+          until @stop
+            @lock.synchronize do
+              flush_messages true
+            end
+            sleep @flush_interval
+          end
+        end
+
+        @thread = Thread.new do
+          until @stop
+            event = JSON::generate(opts[:input].shift)
+            @lock.synchronize do
+              enqueue event
+            end
+
+            log.trace \
+              event: 'publish',
+              raw: event
+          end
+        end
+
+        log.info event: 'output started'
+
+        @thread.join if @foreground
+      end
+
+
+      # Join the Output thread. Effectively only once.
+      def join
+        return if @foreground
+        @foreground = true
+        @thread.join
+      end
+
+
+      # Stop the Output thread. Effectively only once.
+      def stop
+        return if @foreground
+        @foreground = true
+        @thread.kill
+        log.info event: 'output stopped'
+      end
+
+
+    private
+      def log ; @logger end
+
+      def open_uri
+        @http = Net::HTTP.new(@uri.host, @uri.port)
+      end
+
+      def enqueue event
+        @messages << event
+        flush_messages
+      end
+
+      def flush_messages force=false
+        size = @messages.length
+        return if size.zero?
+        if force || size >= @flush_size
+          emit @messages.join("\n")
+          @statz.inc :num_output, size
+          @messages.clear
+        end
+      end
+
+      def emit body
+        request = Net::HTTP::Post.new(@uri)
+        request.body = body
+        @http.request(request)
+      rescue EOFError, Errno::ECONNREFUSED, Errno::EPIPE
+        log.warn event: 'output reconnect'
+        open_uri
+        sleep 1
+        retry
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: franz
 version: !ruby/object:Gem::Version
-  version: 2.1.5
+  version: 2.1.6
 platform: ruby
 authors:
 - Sean Clemmer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-20 00:00:00.000000000 Z
+date: 2015-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slog
@@ -141,9 +141,10 @@ files:
 - lib/franz/input_config.rb
 - lib/franz/metadata.rb
 - lib/franz/output.rb
+- lib/franz/output/device.rb
+- lib/franz/output/http.rb
 - lib/franz/output/kafka.rb
 - lib/franz/output/rabbitmq.rb
-- lib/franz/output/stdout.rb
 - lib/franz/sash.rb
 - lib/franz/stats.rb
 - lib/franz/tail.rb