liquid-ext 3.2.0 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 60c594a5e6540ff912a888069e175df3df538689
-  data.tar.gz: b7aad2df11ab4e395b6273f998328ce33e1cc983
+  metadata.gz: d1323dc768902ea05e69c01cb8548540daca2cb3
+  data.tar.gz: f36b10320132fb182ed9233f6eeb1b28c909a78e
 SHA512:
-  metadata.gz: e32319504d77cb97d76b9f4d0b10f50fe978355d36001f8ca62b33cc30cc98ccb8afcca3669b069f34bdce7f96cadc0865b2081e278ae467a4ed6e480e707324
-  data.tar.gz: 093c9039dde0b4c27323fe984906bd5970c827afded74e9c51ad4f0be8b0c07267bc5161b7842e864dca88fc25602e2991592237e52272a3d40747135ca590e6
+  metadata.gz: 1e6182e393fabdd026dc5447b29e6999cc2857ee82cd71c6c9532a77f6e00ad874f63c5954e93718e71ffffea22d9005429786d813968167a1ec1186d964c1cb
+  data.tar.gz: dd79fbd7bfb4d2f507b711174326a094833042e86b506fe4b715248ad0a73310d99a9f9e4121130c34f59c9e38579e548170d3c570f7fc623c52b82604fea014

data/.rspec

@@ -0,0 +1,2 @@
+--require spec_helper
+--color

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    liquid-ext (3.2.0)
+    liquid-ext (3.3.0)
       activesupport
       erubis
       jmx4r
@@ -11,8 +11,8 @@ PATH
 GEM
   remote: http://rubygems.org/
   specs:
-    activesupport (4.0.3)
-      i18n (~> 0.6, >= 0.6.4)
+    activesupport (4.0.4)
+      i18n (~> 0.6, >= 0.6.9)
       minitest (~> 4.2)
       multi_json (~> 1.3)
       thread_safe (~> 0.1)
@@ -110,7 +110,7 @@ GEM
       atomic (>= 1.1.7, < 2)
     timecop (0.7.1)
     tins (1.0.0)
-    tzinfo (0.3.38)
+    tzinfo (0.3.39)
     yard (0.8.7.3)
 
 PLATFORMS

data/lib/liquid/boot.rb

@@ -24,14 +24,6 @@ if RUBY_PLATFORM == 'java'
   Dir[File.join(File.dirname(__FILE__), '*.jar')].each do |f|
     require f
   end
-
-  # some java libraries cannot be found on maven central, so we load all bundled
-  # jar files here for convenience
-  if defined?(ROOT)
-    Dir[File.join(ROOT, 'jars', '*.jar')].each do |f|
-      require f
-    end
-  end
 end
 
 # load a bunch of common classes here, so we don't have to track and repeat it
@@ -53,11 +45,13 @@ require 'liquid/cli'
 require 'liquid/environment'
 require 'liquid/from_file'
 require 'liquid/hash_helper'
+require 'liquid/health_checks'
 require 'liquid/metrics'
 require 'liquid/server'
 require 'liquid/timing'
 require 'liquid/tracker'
 require 'liquid/transaction_id'
+require 'liquid/trove'
 require 'liquid/zmq'
 
 # configuration callbacks
@@ -119,3 +113,8 @@ $conf.mixin({
 $conf.callback(&reload_mixins)
 $conf.callback(&reload_logger)
 $conf.reload!
+
+if defined?(ROOT) && $conf.code_reloader
+  require 'liquid/code_reloader'
+  CodeReloader.new(ROOT)
+end

data/lib/liquid/code_reloader.rb

@@ -0,0 +1,28 @@
+require 'listen'
+
+class CodeReloader
+
+  def initialize(path)
+    $log.warn("code.reloader", active: true, path: path)
+    Listen.to(path) do |m, a, r|
+      Thread.name = "Code Reloader (#{path})"
+      (m + a).uniq.each do |file|
+        reload(file)
+      end
+    end.start
+  end
+
+  def reload(file)
+    if file =~ /\.rb$/
+      $log.info("code.reloader", reload: file)
+      begin
+        load(file)
+      rescue SyntaxError => e
+        $log.exception(e)
+      rescue => e
+        $log.exception(e)
+      end
+    end
+  end
+
+end

data/lib/liquid/ext/socket.rb

@@ -0,0 +1,10 @@
+class Socket
+  def self.ipaddress
+    hostname = %x(hostname).chomp
+    getaddrinfo(hostname, nil).first[3]
+  end
+
+  def self.fqdn
+    Socket.gethostbyaddr(IPAddr.new(ipaddress).hton)[0]
+  end
+end

data/lib/liquid/ext/string.rb

@@ -8,4 +8,17 @@ class String
       self
     end
   end
+
+  # from: http://rubydoc.info/gems/extlib/0.9.15/String#camel_case-instance_method
+  def camel_case
+    return self if self !~ /_/ && self =~ /[A-Z]+.*/
+    split('_').map{|e| e.capitalize}.join
+  end
+
+  def snake_case
+    return downcase if match(/\A[A-Z]+\z/)
+    gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2').
+        gsub(/([a-z])([A-Z])/, '\1_\2').
+        downcase
+  end
 end

data/lib/liquid/health_checks.rb

@@ -1,20 +1,52 @@
 require 'liquid/logger'
 
-require_relative './metrics-healthchecks-3.0.1.jar'
-java_import 'com.codahale.metrics.health.HealthCheckRegistry'
+class HealthCheck
+  class Result
+    attr_reader :message
+    attr_reader :exception
 
-class HealthChecks
-  @registry = HealthCheckRegistry.new
+    def initialize(healthy, message, exception)
+      @healthy = healthy
+      @message = message
+      @exception = exception
+    end
 
-  def self.registry
-    @registry
+    def healthy?
+      @healthy
+    end
+
+    def to_s
+      "Result{isHealthy=#{healthy?},message=#{message.inspect},exception=#{exception.inspect}}"
+    end
   end
 
-  def self.register(name, handler)
-    registry.register(name, handler)
+  @@checks = {}
+
+  def self.inherited(child)
+    @@checks[child.name.demodulize] = child
   end
 
   def self.run
-    registry.run_health_checks
+    @@checks.inject({}) do |result, (name, handler)|
+      result[name] = handler.new.execute
+      result
+    end
+  end
+
+  def self.poll(interval = 5)
+    loop do
+      @healthy = run.values.all?(&:healthy?)
+      sleep(interval)
+    end
+  end
+
+  def self.healthy?
+    @healthy.nil? ? run.values.all?(&:healthy?) : @healthy
+  end
+
+  def execute
+    check
+  rescue => e
+    Result.new(false, "failed to execute check", e)
   end
 end

data/lib/liquid/logger.rb

@@ -1,27 +1,23 @@
-require 'liquid/ext/string'
-
-require_relative './slf4j-api-1.7.6.jar'
-require_relative './slf4j-log4j12-1.7.6.jar'
-require_relative './log4j-1.2.17.jar'
 java_import 'org.slf4j.LoggerFactory'
 
 module Liquid
   class Logger
 
     attr_accessor :progname
+    attr_accessor :appender
 
     def initialize(name, progname = nil)
       @progname = progname || File.basename($0)
       @logger = LoggerFactory.getLogger(name)
       @exceptions = {}
       @exception_handlers = [method(:_log_error_exception)]
-      reload!
+      unmute!
     end
 
     def reload!
       root = org.apache.log4j.Logger.getRootLogger
-      appender = org.apache.log4j.ConsoleAppender.new
-      appender.name = "console"
+      appender = @appender.new
+      appender.name = "default"
       appender.layout = org.apache.log4j.PatternLayout.new($conf.log.format)
       appender.threshold = org.apache.log4j.Level.toLevel($conf.log.level.to_s)
       appender.activateOptions
@@ -29,6 +25,16 @@ module Liquid
       root.addAppender(appender)
     end
 
+    def mute!
+      @appender = org.apache.log4j.varia.NullAppender
+      reload!
+    end
+
+    def unmute!
+      @appender = org.apache.log4j.ConsoleAppender
+      reload!
+    end
+
     def trace?
       @logger.trace_enabled?
     end

data/lib/liquid/metrics.rb

@@ -1,6 +1,3 @@
-require 'liquid/logger'
-
-require_relative './metrics-core-3.0.1.jar'
 java_import 'com.codahale.metrics.Histogram'
 java_import 'com.codahale.metrics.JmxReporter'
 java_import 'com.codahale.metrics.MetricRegistry'

data/lib/liquid/server.rb

@@ -9,6 +9,7 @@ module Liquid
       initialize_raven
       initialize_tracker
       initialize_metrics
+      initialize_health_checks
     end
 
     def initialize_raven
@@ -43,6 +44,13 @@ module Liquid
       Signal.register_shutdown_handler { ::Metrics.stop }
     end
 
+    def initialize_health_checks
+      Thread.new do
+        Thread.name = "Health Check"
+        HealthCheck.poll
+      end
+    end
+
     def initialize_zmachine
       require 'zmachine'
       ZMachine.logger = $log

data/lib/liquid/tracker/kafka_tracker.rb

@@ -1,11 +1,5 @@
 require 'liquid/tracker/base'
 
-require_relative '../scala-library-2.10.3.jar'
-require_relative '../metrics-core-2.2.0.jar'
-require_relative '../metrics-annotation-2.2.0.jar'
-require_relative '../kafka_2.10-0.8.0.jar'
-require_relative '../snappy-java-1.1.1-M1.jar'
-
 module Tracker
   class KafkaTracker < Base
 

data/lib/liquid/trove.rb

@@ -0,0 +1,71 @@
+java_import 'gnu.trove.TIntCollection'
+java_import 'gnu.trove.set.hash.TIntHashSet'
+java_import 'gnu.trove.set.hash.TLongHashSet'
+java_import 'gnu.trove.map.hash.TIntObjectHashMap'
+java_import 'gnu.trove.map.hash.TLongObjectHashMap'
+
+module TMap
+  def each
+    it = iterator
+    while it.has_next
+      it.advance
+      yield it.key, it.value
+    end
+  end
+end
+
+module TSet
+  def each
+    it = iterator
+    while it.has_next
+      yield it.next
+    end
+  end
+
+  def inspect
+    if size > 1000
+      "#{self.class.name}{too large to display,l=#{size}}"
+    else
+      to_string
+    end
+  end
+end
+
+class TLongObjectHashMap
+  include TMap
+  alias :has_key? :containsKey
+  alias :[] :get
+  alias :[]= :put
+  alias :length :size
+  alias :delete :remove
+end
+
+class TIntObjectHashMap
+  include TMap
+  alias :has_key? :containsKey
+  alias :[] :get
+  alias :[]= :put
+  alias :length :size
+  alias :delete :remove
+end
+
+class TLongHashSet
+  include TSet
+  alias :include? :contains
+  alias :length :size
+  alias :delete :remove
+end
+
+class TIntHashSet
+  include TSet
+  alias :include? :contains
+  alias :length :size
+  alias :delete :remove
+
+  java_alias :concat_ints, :addAll, [TIntCollection.java_class]
+
+  def concat(list)
+    return if !list
+    concat_ints list
+  end
+end

data/lib/liquid/zmq.rb

@@ -1,4 +1,3 @@
-require_relative "./jeromq-0.3.3.jar"
 java_import "org.zeromq.ZContext"
 java_import "org.zeromq.ZFrame"
 java_import "org.zeromq.ZLoop"
@@ -8,11 +7,8 @@ java_import "org.zeromq.ZMQQueue"
 java_import "org.zeromq.ZMsg"
 java_import "org.zeromq.ZThread"
 
-java_import "java.nio.channels.AsynchronousCloseException"
-java_import "java.nio.channels.ClosedChannelException"
-java_import "java.nio.channels.ClosedSelectorException"
-
 class ZContext
+
   def create_socket_with_opts(type, opts = {})
     socket = create_socket(type)
     opts.each do |key, value|
@@ -62,12 +58,33 @@ class ZContext
     instance.destroy_socket(socket)
   end
 
+  # really incredible how many exceptions a simple shutdown can throw all over
+  # the place. if it's one thing ZMQ did never get right it is the shutdown
+  # logic ...
   def self.destroy
     instance.destroy
   rescue Java::JavaLang::IllegalStateException
     # ignore broken shutdown in zeromq
   end
 
+  DestroyExceptions = [
+    Java::JavaNioChannels::AsynchronousCloseException,
+    Java::JavaNioChannels::ClosedChannelException,
+    Java::JavaNioChannels::ClosedSelectorException,
+  ]
+
+  Exceptions = DestroyExceptions + [
+    Java::OrgZeromq::ZMQException,
+    Java::Zmq::ZError::IOException,
+  ]
+
+  def self.destroy_exception?(e)
+    return true if e.is_a?(Java::OrgZeromq::ZMQException) && ZMQ::Error::ETERM.getCode == e.getErrorCode
+    return true if e.is_a?(Java::Zmq::ZError::IOException) && DestroyExceptions.include?(e.cause.class)
+    return true if DestroyExceptions.include?(e.class)
+    return false
+  end
+
   def self.router(opts = {})
     instance.router(opts)
   end

data/liquid-ext.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "liquid-ext"
-  spec.version       = "3.2.0"
+  spec.version       = "3.3.0"
   spec.authors       = ["LiquidM, Inc."]
   spec.email         = ["opensource@liquidm.com"]
   spec.description   = %q{Ruby core extensions and helper libraries}
@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "activesupport"
   spec.add_dependency "erubis"
+  spec.add_dependency "jmx4r"
   spec.add_dependency "mixlib-cli"
   spec.add_dependency "terminal-table"
-  spec.add_dependency "jmx4r"
 end

data/spec/spec_helper.rb

@@ -1,10 +1,20 @@
-require 'rubygems'
-require 'rspec'
+ROOT = File.expand_path('../..', __FILE__)
 
-require 'simplecov'
-SimpleCov.start
+require 'bundler/setup'
+require 'liquid/boot'
 
-$:.unshift(File.dirname(__FILE__) + '/../lib')
+require 'rspec'
 
 RSpec.configure do |config|
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
 end
+
+# load helper classes & functions from files in spec/support/
+Dir[File.join(ROOT, "spec/support/**/*.rb")].each { |f| require f }
+
+# be silent
+$log.mute!

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: liquid-ext
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 3.3.0
 platform: ruby
 authors:
 - LiquidM, Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-03-11 00:00:00.000000000 Z
+date: 2014-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -39,7 +39,7 @@ dependencies:
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
-  name: mixlib-cli
+  name: jmx4r
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -53,7 +53,7 @@ dependencies:
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
-  name: terminal-table
+  name: mixlib-cli
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -67,7 +67,7 @@ dependencies:
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
-  name: jmx4r
+  name: terminal-table
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -89,6 +89,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .rspec
 - .ruby-version
 - .travis.yml
 - .yardopts
@@ -101,8 +102,14 @@ files:
 - lib/liquid/benchmark.rb
 - lib/liquid/boot.rb
 - lib/liquid/cli.rb
+- lib/liquid/code_reloader.rb
+- lib/liquid/commons-codec-1.9.jar
+- lib/liquid/commons-io-2.4.jar
 - lib/liquid/configuration.rb
 - lib/liquid/console.rb
+- lib/liquid/curator-client-2.4.1.jar
+- lib/liquid/curator-framework-2.4.1.jar
+- lib/liquid/curator-recipes-2.4.1.jar
 - lib/liquid/environment.rb
 - lib/liquid/ext/array.rb
 - lib/liquid/ext/enumerable.rb
@@ -112,6 +119,7 @@ files:
 - lib/liquid/ext/nil.rb
 - lib/liquid/ext/ordered_set.rb
 - lib/liquid/ext/signal.rb
+- lib/liquid/ext/socket.rb
 - lib/liquid/ext/string.rb
 - lib/liquid/ext/thread.rb
 - lib/liquid/from_file.rb
@@ -122,16 +130,15 @@ files:
 - lib/liquid/kafka_2.10-0.8.0.jar
 - lib/liquid/log4j-1.2.17.jar
 - lib/liquid/logger.rb
+- lib/liquid/mapdb-0.9.3.jar
 - lib/liquid/metrics-annotation-2.2.0.jar
 - lib/liquid/metrics-core-2.2.0.jar
 - lib/liquid/metrics-core-3.0.1.jar
-- lib/liquid/metrics-healthchecks-3.0.1.jar
 - lib/liquid/metrics.rb
 - lib/liquid/metrics/logger_reporter.rb
 - lib/liquid/metrics/reporter.rb
 - lib/liquid/metrics/tracker_reporter.rb
 - lib/liquid/priority_selector.rb
-- lib/liquid/router.rb
 - lib/liquid/scala-library-2.10.3.jar
 - lib/liquid/server.rb
 - lib/liquid/slf4j-api-1.7.6.jar
@@ -140,9 +147,7 @@ files:
 - lib/liquid/tasks.rb
 - lib/liquid/tasks/protoc.rake
 - lib/liquid/tasks/reek.rake
-- lib/liquid/tasks/router.rake
 - lib/liquid/tasks/rspec.rake
-- lib/liquid/tasks/yard.rake
 - lib/liquid/templates/Gemfile.tt
 - lib/liquid/templates/LICENSE.txt.tt
 - lib/liquid/templates/README.md.tt
@@ -162,11 +167,12 @@ files:
 - lib/liquid/tracker/logger_tracker.rb
 - lib/liquid/transaction_id.rb
 - lib/liquid/trove-3.0.3.jar
+- lib/liquid/trove.rb
 - lib/liquid/weighted_selector.rb
 - lib/liquid/zmq.rb
+- lib/liquid/zookeeper-3.4.6.jar
 - liquid-ext.gemspec
 - spec/lib/liquid/ext/enumerable_spec.rb
-- spec/lib/liquid/router_spec.rb
 - spec/lib/liquid/weighted_selector_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/liquidm/ext
@@ -195,7 +201,6 @@ specification_version: 4
 summary: Ruby core extensions and helper libraries
 test_files:
 - spec/lib/liquid/ext/enumerable_spec.rb
-- spec/lib/liquid/router_spec.rb
 - spec/lib/liquid/weighted_selector_spec.rb
 - spec/spec_helper.rb
 has_rdoc:

data/lib/liquid/router.rb

@@ -1,45 +0,0 @@
-# encoding: utf-8
-
-require 'liquid/from_file'
-
-class Router
-  include FromFile
-
-  attr_accessor :routes
-
-  def initialize(request_handler)
-    @request_handler = request_handler
-    @routes = []
-
-    @cache = Hash.new do |hash, path|
-      hash[path] = nil
-      @routes.each do |regex, block, args|
-        match = path.match(regex)
-        hash[path] = block.curry[match] if match
-      end
-      hash[path]
-    end
-  end
-
-  def add(regexp, args, &block)
-    @routes << [regexp, block, args]
-  end
-
-  # route %r(/foo/(.+)/(\w+)/(\d+)), AnyParser, :matches, :in, :order
-  def route(regexp, parser, *args, &block)
-    block = lambda do |match, env|
-      params = args.each_with_index.inject({}) do |hash, (name, index)|
-        hash[name] = match[index+1]
-        hash
-      end
-
-      return @request_handler.handle(parser, env, params)
-    end
-
-    add(regexp, args, &block)
-  end
-
-  def handle(path, request)
-    @cache[path].call(request) if @cache[path]
-  end
-end

data/lib/liquid/tasks/router.rake

@@ -1,17 +0,0 @@
-# encoding: utf-8
-
-require 'liquid/router'
-require 'terminal-table'
-
-desc "Show all ed routes"
-task :routes do
-  root = File.expand_path(File.dirname(__FILE__))
-  router = Router.new(lambda{})
-  router.from_file(File.join(root, "../../../config/routes.rb"))
-  routes = router.routes.map {|r| [r[0].inspect, r[1], r[2] ] }
-  puts Terminal::Table.new({
-    rows: routes.sort_by { |route| route[0] },
-    headings: ['regex', 'handler', 'parameters'],
-  })
-end
-

data/lib/liquid/tasks/yard.rake

@@ -1,8 +0,0 @@
-begin
-  require 'yard'
-
-  YARD::Rake::YardocTask.new do |t|
-    t.files = ['lib/**/*.rb', 'README.rdoc']
-  end
-rescue LoadError
-end

data/spec/lib/liquid/router_spec.rb

@@ -1,75 +0,0 @@
-# encoding: utf-8
-
-require 'liquid/router'
-
-class GeneralRequestHandler; end
-
-describe Router do
-  let(:regex) { %r(foo(.*)bar) }
-  let(:block) { Proc.new { 1+1 } }
-  let(:router) { Router.new(GeneralRequestHandler) }
-
-  describe '.add' do
-    context 'add route' do
-
-      before { router.add(regex, [:args], &block) }
-
-      context 'regex' do
-        subject { router.routes.last[0] }
-        it { should eql regex }
-      end
-
-      context 'block' do
-        subject { router.routes.last[1] }
-        its(:call) { should eql block.call }
-      end
-
-      context 'args' do
-        subject { router.routes.last[2] }
-        it { should eql [:args] }
-      end
-    end
-  end
-
-  describe '.route' do
-    let(:regex) { %r(/bar/) }
-
-    let(:parser) { double('CustomRequestParser') }
-    let(:env)    { "env" }
-
-    before { router.route(regex, parser, :foo, :bar) }
-
-    subject { router.routes.last[0] }
-
-    it { should be_a Regexp }
-    it { should eql regex }
-    it 'builds correct block' do
-      GeneralRequestHandler.should_receive(:handle).with(parser, env, {foo: 1, bar: 2})
-
-      _, block = router.routes.last
-      block.call([0,1,2], env)
-    end
-  end
-
-  describe '.handle' do
-    let(:handler) { double('MockHandler') }
-    let(:request) { double('request') }
-    let(:url_re_1) { 'http://ad.madvertise.de/muchte/(.*?)' }
-    let(:block) { Proc.new { |match, env| handler.handle(match, env) } }
-
-    before do
-      router.add url_re_1, [:foo], &block
-      router.route %r(/new_adx_bidrequest/([\w.-]+)), GeneralRequestHandler, :site_token
-    end
-
-    it 'is not matching' do
-      router.handle("http://ad.madvertise.de/muchtel", request).should == nil
-    end
-
-    it 'is matching' do
-      handler.should_receive(:handle).and_return("bar")
-      router.handle("http://ad.madvertise.de/muchte/foo", request).should == "bar"
-    end
-  end
-end
-