liquid-ext 2.0.3 → 3.0.0

data/lib/liquid/logger.rb

@@ -0,0 +1,122 @@
+require 'liquid/ext/string'
+
+require_relative './slf4j-api-1.7.6.jar'
+require_relative './slf4j-log4j12-1.7.6.jar'
+require_relative './log4j-1.2.17.jar'
+java_import 'org.slf4j.LoggerFactory'
+
+module Liquid
+  class Logger
+
+    attr_accessor :progname
+
+    def initialize(name, progname = nil)
+      @progname = progname || File.basename($0)
+      @logger = LoggerFactory.getLogger(name)
+      @exceptions = {}
+      root = org.apache.log4j.Logger.getRootLogger
+      appender = org.apache.log4j.ConsoleAppender.new
+      appender.name = "console"
+      appender.layout = org.apache.log4j.PatternLayout.new($conf.log.format)
+      appender.threshold = org.apache.log4j.Level.toLevel($conf.log.level.to_s)
+      appender.activateOptions
+      root.removeAllAppenders
+      root.addAppender(appender)
+    end
+
+    def trace?
+      @logger.trace_enabled?
+    end
+
+    def trace(*args, &block)
+      return unless trace?
+      args = yield if block_given?
+      @logger.trace(format(*args))
+    end
+
+    def debug?
+      @logger.debug_enabled?
+    end
+
+    def debug(*args, &block)
+      return unless debug?
+      args = yield if block_given?
+      @logger.debug(format(*args))
+    end
+
+    def info?
+      @logger.info_enabled?
+    end
+
+    def info(*args, &block)
+      return unless info?
+      args = yield if block_given?
+      @logger.info(format(*args))
+    end
+
+    def warn?
+      @logger.warn_enabled?
+    end
+
+    def warn(*args, &block)
+      return unless warn?
+      args = yield if block_given?
+      @logger.warn(format(*args))
+    end
+
+    def error?
+      @logger.error_enabled?
+    end
+
+    def error(*args, &block)
+      return unless error?
+      args = yield if block_given?
+      @logger.error(format(*args))
+    end
+
+    def exception(exc, message = nil, attribs = {})
+      ::Metrics.meter("exception:#{exc.class.to_s.tableize}").mark
+      @exceptions[exc.class] ||= {}
+      @exceptions[exc.class][exc.backtrace.first] ||= [System.nano_time, 1, 1]
+      five_minutes_ago = System.nano_time - 300_000_000_000
+      last, count, backoff = *@exceptions[exc.class][exc.backtrace.first]
+      count = backoff = 1 if last < five_minutes_ago
+      backoff = count > backoff ? backoff * 2 : backoff
+      if count % backoff == 0
+        error("exception", {
+          class: exc.class,
+          count: count,
+          reason: exc.message,
+          message: message,
+          backtrace: exc.backtrace
+        }.merge(attribs).merge(called_from))
+      end
+      @exceptions[exc.class][exc.backtrace.first] = [
+        System.nano_time,
+        count + 1,
+        backoff
+      ]
+    end
+
+    private
+
+    def format(message, attribs = {})
+      attribs.merge!(called_from) if $conf.log.caller
+      attribs = attribs.map do |k,v|
+        "#{k}=#{v.to_s.clean_quote}"
+      end.join(' ')
+      message += " #{attribs}" if attribs.length > 0
+      message
+    end
+
+    # Return the first callee outside the liquid-ext gem
+    def called_from
+      location = caller.detect('unknown:0') do |line|
+        line.match(/\/liquid(-|\/)ext/).nil?
+      end
+      file, line, _ = location.split(':')
+      { :file => file, :line => line }
+    end
+
+  end
+end

data/lib/liquid/metrics/logger_reporter.rb

@@ -3,13 +3,10 @@ require 'liquid/metrics/reporter'
 module Metrics
   class LoggerReporter < Reporter
 
-    attr_accessor :logger
-    attr_accessor :marker
-
-    def initialize(registry)
+    def initialize(logger)
       super
-      @logger = $log
-      @marker = "metriks:"
+      @logger = logger
+      @marker = "metrics:"
     end
 
     def report_gauge(name, gauge)

data/lib/liquid/metrics/reporter.rb

@@ -11,12 +11,13 @@ module Metrics
     attr_accessor :rate_unit
     attr_accessor :duration_unit
 
-    def initialize(registry)
-      @registry = registry
+    def initialize(*args)
+      @registry = ::Metrics.registry
       @filter = MetricFilter::ALL
       @executor = Executors.newSingleThreadScheduledExecutor
       self.rate_unit = TimeUnit::SECONDS
       self.duration_unit = TimeUnit::MILLISECONDS
+      ::Metrics.register_reporter(self)
     end
 
     def rate_unit=(value)
@@ -39,9 +40,7 @@ module Metrics
       $log.exception(e)
     end
 
-    def start(period = nil, unit = nil)
-      period ||= 5
-      unit ||= TimeUnit::MINUTES
+    def start(period = 60, unit = TimeUnit::SECONDS)
       @executor.scheduleAtFixedRate(self, period, period, unit)
     end
 
@@ -82,6 +81,7 @@ module Metrics
 
     def report_gauge(name, gauge)
       {
+        timestamp: Time.now.to_i,
         type: :gauge,
         name: name,
         value: gauge.value,
@@ -90,6 +90,7 @@ module Metrics
 
     def report_counter(name, counter)
       {
+        timestamp: Time.now.to_i,
         type: :counter,
         name: name,
         count: counter.count,
@@ -99,6 +100,7 @@ module Metrics
     def report_histogram(name, histogram)
       snapshot = histogram.snapshot
       {
+        timestamp: Time.now.to_i,
         type: :histogram,
         name: name,
         count: histogram.count,
@@ -107,25 +109,31 @@ module Metrics
         mean: snapshot.getMean,
         stdev: snapshot.getStdDev,
         median: snapshot.getMedian,
-        :'95th_percentile' => snapshot.get95thPercentile,
+        p75: snapshot.get75thPercentile,
+        p95: snapshot.get95thPercentile,
+        p98: snapshot.get98thPercentile,
+        p99: snapshot.get99thPercentile,
+        p999: snapshot.get999thPercentile,
       }
     end
 
     def report_meter(name, meter)
       {
+        timestamp: Time.now.to_i,
         type: :meter,
         name: name,
         count: meter.count,
         mean_rate: convert_rate(meter.getMeanRate),
-        one_minute_rate: convert_rate(meter.getOneMinuteRate),
-        five_minute_rate: convert_rate(meter.getFiveMinuteRate),
-        fifteen_minute_rate: convert_rate(meter.getFifteenMinuteRate),
+        m1: convert_rate(meter.getOneMinuteRate),
+        m5: convert_rate(meter.getFiveMinuteRate),
+        m15: convert_rate(meter.getFifteenMinuteRate),
       }
     end
 
     def report_timer(name, timer)
       snapshot = timer.snapshot
       {
+        timestamp: Time.now.to_i,
         type: :timer,
         name: name,
         min: convert_duration(snapshot.getMin),
@@ -133,11 +141,15 @@ module Metrics
         mean: convert_duration(snapshot.getMean),
         stdev: convert_duration(snapshot.getStdDev),
         median: convert_duration(snapshot.getMedian),
-        :'95th_percentile' => convert_duration(snapshot.get95thPercentile),
+        p75: convert_duration(snapshot.get75thPercentile),
+        p95: convert_duration(snapshot.get95thPercentile),
+        p98: convert_duration(snapshot.get98thPercentile),
+        p99: convert_duration(snapshot.get99thPercentile),
+        p999: convert_duration(snapshot.get999thPercentile),
         mean_rate: convert_rate(timer.getMeanRate),
-        one_minute_rate: convert_rate(timer.getOneMinuteRate),
-        five_minute_rate: convert_rate(timer.getFiveMinuteRate),
-        fifteen_minute_rate: convert_rate(timer.getFifteenMinuteRate),
+        m1: convert_rate(timer.getOneMinuteRate),
+        m5: convert_rate(timer.getFiveMinuteRate),
+        m15: convert_rate(timer.getFifteenMinuteRate),
       }
     end
 

data/lib/liquid/metrics/tracker_reporter.rb

@@ -0,0 +1,33 @@
+require 'liquid/metrics/reporter'
+require 'liquid/tracker'
+
+module Metrics
+  class TrackerReporter < Reporter
+
+    def initialize(tracker)
+      super
+      @tracker = tracker
+    end
+
+    def report_gauge(name, gauge)
+      @tracker.event(super)
+    end
+
+    def report_counter(name, counter)
+      @tracker.event(super)
+    end
+
+    def report_histogram(name, histogram)
+      @tracker.event(super)
+    end
+
+    def report_meter(name, meter)
+      @tracker.event(super)
+    end
+
+    def report_timer(name, timer)
+      @tracker.event(super)
+    end
+
+  end
+end

data/lib/liquid/metrics.rb

@@ -1,42 +1,41 @@
-require 'singleton'
+require 'liquid/logger'
 
+require_relative './metrics-core-3.0.1.jar'
 java_import 'com.codahale.metrics.Histogram'
-java_import 'com.codahale.metrics.MetricRegistry'
 java_import 'com.codahale.metrics.JmxReporter'
-
-require 'liquid/metrics/logger_reporter'
+java_import 'com.codahale.metrics.MetricRegistry'
+java_import 'java.util.concurrent.TimeUnit'
 
 class Histogram
   java_alias :update_long, :update, [Java::long]
 end
 
 module Metrics
-
-  @@registry = MetricRegistry.new
-  @@reporters = []
-
-  def self.start
-    @@reporters << JmxReporter.forRegistry(@@registry).build
-    @@reporters << LoggerReporter.new(@@registry)
-
-    @@reporters.each(&:start)
-    Signal.register_shutdown_handler { self.stop }
+  @registry = MetricRegistry.new
+  @reporters = []
+
+  def self.start(period = nil, unit = nil)
+    @period ||= 60
+    @unit ||= TimeUnit::SECONDS
+    JmxReporter.forRegistry(@registry).build.start
+    Signal.register_shutdown_handler { stop }
   end
 
   def self.stop
-    @@reporters.each do |reporter|
-      if reporter.respond_to?(:run)
-        reporter.run
-      end
-
+    @reporters.each do |reporter|
+      reporter.run if reporter.respond_to?(:run)
       reporter.stop
     end
+    @reporters.clear
+  end
 
-    @@reporters.clear
+  def self.register_reporter(reporter)
+    reporter.start(@period, @unit)
+    @reporters << reporter
   end
 
   def self.registry
-    @@registry
+    @registry
   end
 
   def self.counter(name)

data/lib/liquid/router.rb

@@ -41,6 +41,5 @@ class Router
 
   def handle(path, request)
     @cache[path].call(request) if @cache[path]
-    return !! @cache[path]
   end
 end

data/lib/liquid/templates/config.yml.tt

@@ -1 +0,0 @@
-log_level: INFO

data/lib/liquid/timing.rb

@@ -15,24 +15,16 @@ module Timing
     end
 
     def tick
-      rt = runtime_since(@last_tick)
+      rt = System.nano_time - @last_tick
       @last_tick = System.nano_time
       rt
     end
 
     def stop
-      rt = runtime_since(@start)
+      rt = System.nano_time - @start
       reset!
       rt
     end
 
-    private
-
-    def runtime_since(start)
-      rt = System.nano_time - start
-      rt = rt.to_f / 1_000_000_000
-      rt
-    end
-
   end
 end

data/lib/liquid/tracker/base.rb

@@ -0,0 +1,26 @@
+require 'liquid/tracker/json_serializer'
+
+module Tracker
+  class Base
+    attr_accessor :serializer
+
+    def initialize(dimensions = {})
+      @serializer = JsonSerializer.new(dimensions)
+    end
+
+    def with_topic(topic)
+      Topic.new(topic, self)
+    end
+  end
+
+  class Topic
+    def initialize(topic, tracker)
+      @topic = topic
+      @tracker = tracker
+    end
+
+    def event(obj)
+      @tracker.event(obj, @topic)
+    end
+  end
+end

data/lib/liquid/tracker/blackhole_tracker.rb

@@ -0,0 +1,15 @@
+require 'liquid/tracker/base'
+
+module Tracker
+  class BlackholeTracker < Base
+    def event(obj, topic)
+    end
+
+    def down?
+      false
+    end
+
+    def shutdown
+    end
+  end
+end

data/lib/liquid/tracker/file_tracker.rb

@@ -0,0 +1,21 @@
+require 'liquid/tracker/base'
+
+module Tracker
+  class FileTracker < Base
+
+    def event(obj, topic)
+      file = File.open(File.join(ROOT, 'log', "tracker-#{topic}.log"), 'a')
+      file.sync = true
+      file.write(@serializer.dump(obj))
+      file.write("\n")
+      file.close
+    end
+
+    def down?
+      false
+    end
+
+    def shutdown
+    end
+  end
+end

data/lib/liquid/tracker/json_serializer.rb

@@ -0,0 +1,13 @@
+require 'multi_json'
+
+module Tracker
+  class JsonSerializer
+    def initialize(dimensions)
+      @dimensions = dimensions
+    end
+
+    def dump(obj)
+      MultiJson.dump(@dimensions.merge(obj))
+    end
+  end
+end

data/lib/liquid/tracker/kafka_tracker.rb

@@ -0,0 +1,41 @@
+require 'liquid/tracker/base'
+
+require_relative '../scala-library-2.10.3.jar'
+require_relative '../metrics-core-2.2.0.jar'
+require_relative '../metrics-annotation-2.2.0.jar'
+require_relative '../kafka_2.10-0.8.0.jar'
+require_relative '../snappy-java-1.1.1-M1.jar'
+
+module Tracker
+  class KafkaTracker < Base
+
+    java_import 'kafka.javaapi.producer.Producer'
+    java_import 'kafka.producer.ProducerConfig'
+    java_import 'kafka.producer.KeyedMessage'
+
+    def initialize(properties, dimensions = {})
+      super(dimensions)
+      @producer = Producer.new(ProducerConfig.new(properties))
+    end
+
+    def down?
+      # TODO: async is fire and forget. we might want to handle
+      # QueueFullExceptions later
+      false
+    end
+
+    def event(obj, topic)
+      @producer.send(KeyedMessage.new(topic, @serializer.dump(obj)))
+    rescue => e
+      # TODO: maybe fall back to FileTracker here
+      $log.exception(e, "failed to log event=#{obj.inspect}")
+    end
+
+    def shutdown
+      @producer.close if @producer
+    rescue Java::KafkaProducer::ProducerClosedException
+      # pass
+    end
+
+  end
+end

data/lib/liquid/tracker/logger_tracker.rb

@@ -0,0 +1,17 @@
+require 'liquid/boot'
+require 'liquid/tracker/base'
+
+module Tracker
+  class LoggerTracker < Base
+    def event(obj, topic)
+      $log.info("tracker:event:#{topic} #{@serializer.dump(obj)}")
+    end
+
+    def down?
+      false
+    end
+
+    def shutdown
+    end
+  end
+end

data/lib/liquid/tracker.rb

@@ -0,0 +1,14 @@
+require 'liquid/metrics'
+require 'liquid/metrics/tracker_reporter'
+
+module Tracker
+  class Metrics
+    def self.event
+      (@events ||= ::Metrics.meter("tracker.events")).mark
+    end
+  end
+end
+
+Dir[File.expand_path("../tracker/*.rb", __FILE__)].each do |f|
+  require f
+end

data/lib/liquid/transaction_id.rb

@@ -6,8 +6,8 @@ class TransactionId
     @@current
   end
 
-  def self.next
-    @@current = "#{Time.now.to_i}-#{Process.pid}-#{rand(89999) + 10000}"
+  def self.next(seed = 10000)
+    @@current = "#{Time.now.to_i}-#{Process.pid}-#{rand(89999) + seed}"
   end
 
 end

data/lib/liquid/zmq.rb

@@ -0,0 +1,89 @@
+require_relative "./jeromq-0.3.3.jar"
+java_import "org.zeromq.ZContext"
+java_import "org.zeromq.ZFrame"
+java_import "org.zeromq.ZLoop"
+java_import "org.zeromq.ZMQ"
+java_import "org.zeromq.ZMQException"
+java_import "org.zeromq.ZMQQueue"
+java_import "org.zeromq.ZMsg"
+java_import "org.zeromq.ZThread"
+java_import "java.nio.channels.ClosedSelectorException"
+
+class ZContext
+  def create_socket_with_opts(type, opts = {})
+    socket = create_socket(type)
+    opts.each do |key, value|
+      next if key == :bind || key == :connect
+      socket.__send__("#{key}=", value)
+    end
+    socket.connect(opts[:connect]) if opts[:connect]
+    socket.bind(opts[:bind]) if opts[:bind]
+    socket
+  end
+
+  def router(opts = {})
+    create_socket_with_opts(ZMQ::ROUTER, opts)
+  end
+
+  def dealer(opts = {})
+    create_socket_with_opts(ZMQ::DEALER, opts)
+  end
+
+  def push(opts = {})
+    create_socket_with_opts(ZMQ::PUSH, opts)
+  end
+
+  def pull(opts = {})
+    create_socket_with_opts(ZMQ::PULL, opts)
+  end
+
+  def pub(opts = {})
+    create_socket_with_opts(ZMQ::PUB, opts)
+  end
+
+  def sub(opts = {})
+    create_socket_with_opts(ZMQ::SUB, opts)
+  end
+
+  ## global context instance
+
+  def self.instance
+    @context ||= new
+  end
+
+  def self.create_socket(type)
+    instance.create_socket(type)
+  end
+
+  def self.destroy_socket(socket)
+    instance.destroy_socket(socket)
+  end
+
+  def self.destroy
+    instance.destroy
+  end
+
+  def self.router(opts = {})
+    instance.router(opts)
+  end
+
+  def self.dealer(opts = {})
+    instance.dealer(opts)
+  end
+
+  def self.push(opts = {})
+    instance.push(opts)
+  end
+
+  def self.pull(opts = {})
+    instance.pull(opts)
+  end
+
+  def self.pub(opts = {})
+    instance.pub(opts)
+  end
+
+  def self.sub(opts = {})
+    instance.sub(opts)
+  end
+end

data/liquid-ext.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "liquid-ext"
-  spec.version       = "2.0.3"
+  spec.version       = "3.0.0"
   spec.authors       = ["LiquidM, Inc."]
   spec.email         = ["opensource@liquidm.com"]
   spec.description   = %q{Ruby core extensions and helper libraries}
@@ -17,7 +17,6 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "activesupport"
   spec.add_dependency "erubis"
-  spec.add_dependency "liquid-logging", ">= 2.0.0"
   spec.add_dependency "mixlib-cli"
   spec.add_dependency "terminal-table"
 end