jruby-druid 1.0.0.pre.rc4 → 2.0.0.edge.1

data/lib/druid/logger.rb

@@ -1,81 +0,0 @@
-# Supported Levels: https://github.com/twitter/util
-require 'logger'
-
-module Druid
-  class Logger
-    include Druid::TopLevelPackages
-
-    attr_accessor :logger
-    delegate :debug, :info, :warn, :error, :fatal, :unknown, to: :logger
-
-    def initialize
-      @logger = ::Logger.new(STDOUT)
-    end
-
-    def set_level(level)
-      set_logger_level(level)
-      set_logback_level(level)
-      set_twitter_level(level)
-    end
-
-    private
-
-    def set_logback_level(level)
-      logger = org.slf4j.impl.StaticLoggerBinder.getSingleton.getLoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME)
-      logger.setLevel(get_logback_level(level))
-    end
-
-    def set_logger_level(level)
-      logger.level = get_logger_level(level)
-    end
-
-    def set_twitter_level(level)
-      com.twitter.logging.Logger.
-        get("com.twitter").
-        setLevel(java.util.logging.Level::WARNING)
-    end
-
-    def get_logback_level(level)
-      ch.qos.logback.classic.Level.const_get(map_logback_level(level))
-    end
-
-    def get_logger_level(level)
-      @logger.class.const_get(map_logger_level(level))
-    end
-
-    def get_twitter_level(level)
-      java.util.logging.Level.const_get(map_twitter_level(level))
-    end
-
-    def map_logback_level(level)
-      case level
-      when :critical
-        'ERROR'.freeze
-      when :fatal
-        'ERROR'.freeze
-      else
-        level.to_s.upcase
-      end
-    end
-
-    def map_logger_level(level)
-      case level
-      when :trace
-        'DEBUG'.freeze
-      when :critical
-        'ERROR'.freeze
-      else
-        level.to_s.upcase
-      end
-    end
-
-    def map_twitter_level(level)
-      case level
-      when :warn
-        'WARNING'.freeze
-      else
-        level.to_s.upcase
-      end
-    end
-  end
-end

data/lib/druid/logging.rb

@@ -1,7 +0,0 @@
-module Druid
-  module Logging
-    def logger
-      @@logger ||= Druid::Logger.new
-    end
-  end
-end

data/lib/druid/top_level_packages.rb

@@ -1,11 +0,0 @@
-module Druid
-  module TopLevelPackages
-    def ch
-      Java::Ch
-    end
-
-    def io
-      Java::Io
-    end
-  end
-end

data/lib/druid/writer/base.rb

@@ -1,59 +0,0 @@
-module Druid
-  module Writer
-    class Base
-      attr_accessor :tranquilizers
-      attr_reader :config
-
-      def initialize(config)
-        @config = config
-        @tranquilizers = []
-      end
-
-      def remove_tranquilizer_for_datasource(datasource)
-        tranquilizer = tranquilizer_for_datasource(datasource)
-        remove_tranquilizer(tranquilizer) if tranquilizer
-      end
-
-      def write_point(datasource, datapoint)
-        datapoint = Druid::Writer::Tranquilizer::Datapoint.new(datapoint)
-        sender = get_tranquilizer(datasource, datapoint)
-        sender.safe_send(datapoint)
-      end
-
-      private
-
-      def build_tranquilizer(datasource, datapoint)
-        Druid::Writer::Tranquilizer::Base.new({config: config, datasource: datasource, datapoint: datapoint})
-      end
-
-      def get_tranquilizer(datasource, datapoint)
-        tranquilizer = tranquilizer_for_datasource(datasource)
-
-        unless has_current_schema?(tranquilizer, datapoint)
-          remove_tranquilizer(tranquilizer) if tranquilizer
-          tranquilizer = build_tranquilizer(datasource, datapoint)
-          tranquilizers << tranquilizer
-        end
-
-        tranquilizer
-      end
-
-      def has_current_schema?(tranquilizer, datapoint)
-        return false unless tranquilizer
-        dimensions = tranquilizer.rollup.dimensions.specMap["dimensions"].to_a
-        aggregators = tranquilizer.rollup.aggregators
-        metrics = Java::ScalaCollection::JavaConverters.seqAsJavaListConverter(aggregators).asJava.to_a.map{ |metric| metric.getFieldName }
-        dimensions == datapoint.dimensions.keys && metrics == datapoint.metrics.keys
-      end
-
-      def remove_tranquilizer(tranquilizer)
-        tranquilizers.delete(tranquilizer)
-        tranquilizer.stop
-      end
-
-      def tranquilizer_for_datasource(datasource)
-        tranquilizers.detect{ |t| t.datasource == datasource }
-      end
-    end
-  end
-end

data/lib/druid/writer/tranquilizer.rb

@@ -1,16 +0,0 @@
-module Druid
-  module Writer
-    module Tranquilizer
-      extend Druid::TopLevelPackages
-
-      java_import com.google.common.collect.ImmutableList
-      java_import com.google.common.collect.ImmutableMap
-      java_import com.metamx.tranquility.beam.ClusteredBeamTuning
-      java_import io.druid.query.aggregation.LongSumAggregatorFactory
-      java_import io.druid.granularity.QueryGranularity
-      java_import io.druid.data.input.impl.TimestampSpec
-      java_import org.apache.curator.framework.CuratorFrameworkFactory
-      java_import Java::ScalaCollection::JavaConverters
-    end
-  end
-end

data/lib/druid/writer/tranquilizer/aggregators.rb

@@ -1,15 +0,0 @@
-module Druid
-  module Writer
-    module Tranquilizer
-      class Aggregators
-        class << self
-          def build(metrics)
-            aggs = []
-            metrics.keys.each{ |metric_name| aggs << LongSumAggregatorFactory.new(metric_name, metric_name) }
-            ImmutableList.of(*aggs)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/druid/writer/tranquilizer/base.rb

@@ -1,76 +0,0 @@
-# Based on Java Example below
-# https://github.com/gianm/tranquility-example/blob/master/src/main/java/io/imply/tranquility/JavaExample.java
-
-module Druid
-  module Writer
-    module Tranquilizer
-      class Base
-        attr_reader :config,
-                    :curator,
-                    :datasource,
-                    :rollup,
-                    :service,
-                    :tuning
-
-        def initialize(params)
-          @config = params[:config]
-          @datasource = params[:datasource].to_s
-          @rollup = Druid::Writer::Tranquilizer::Rollup.build(config, params[:datapoint])
-          @tuning = Druid::Writer::Tranquilizer::Tuning.build(config)
-          @curator = Druid::Writer::Tranquilizer::Curator.build(config)
-          @service = create_service
-
-          start
-        end
-
-        def safe_send(datapoint)
-          thread = Thread.new{ send(datapoint) }
-          result = thread.join(@config.wait_time)
-          raise Druid::ConnectionError, 'Error connecting to ZooKeeper' unless result
-          Druid::Writer::Tranquilizer::Future.new(result.value)
-        end
-
-        def start
-          curator.start
-          service.start
-        end
-
-        def stop
-          service.flush
-          service.stop
-          curator.close
-        end
-
-        private
-
-        def argument_map(datapoint)
-          args = datapoint.timestamp.
-            merge(datapoint.dimensions).
-            merge(datapoint.metrics)
-
-          ImmutableMap.builder.putAll(args).build
-        end
-
-        def create_service
-          timestamper = Timestamper.new
-          timestamp_spec = TimestampSpec.new("timestamp", "auto", nil)
-
-          builder = DruidBeams.
-            timestamper_builder(timestamper).
-            curator(curator).
-            discoveryPath(config.discovery_path).
-            location(com.metamx.tranquility.druid.DruidLocation.create(config.index_service, datasource)).
-            timestampSpec(timestamp_spec).
-            rollup(rollup).
-            tuning(tuning)
-          builder = builder.druidBeamConfig(DruidBeamConfig.build(true)) if config.strong_delete
-          builder.buildTranquilizer
-        end
-
-        def send(datapoint)
-          service.send(argument_map(datapoint)).addEventListener(EventListener.new)
-        end
-      end
-    end
-  end
-end

data/lib/druid/writer/tranquilizer/curator.rb

@@ -1,18 +0,0 @@
-module Druid
-  module Writer
-    module Tranquilizer
-      class Curator
-        class << self
-          def build(config)
-            bounded_retry = org.apache.curator.retry.BoundedExponentialBackoffRetry.new(100, 30000, 29)
-            CuratorFrameworkFactory.
-              builder.
-              connectString(config.curator_uri).
-              retryPolicy(bounded_retry).
-              build
-          end
-        end
-      end
-    end
-  end
-end

data/lib/druid/writer/tranquilizer/datapoint.rb

@@ -1,33 +0,0 @@
-module Druid
-  module Writer
-    module Tranquilizer
-      class Datapoint
-        TIMESTAMP_LABEL = 'timestamp'.freeze
-
-        attr_reader :dimensions, :timestamp, :metrics
-
-        def initialize(datapoint)
-          @timestamp = build_time(datapoint[:timestamp])
-          @dimensions = parse_dimensions(datapoint[:dimensions])
-          @metrics = parse_metrics(datapoint[:metrics])
-        end
-
-        private
-
-        def build_time(time)
-          time = Time.now.utc unless time
-          Hash[TIMESTAMP_LABEL, time.iso8601]
-        end
-
-        def parse_dimensions(dimensions)
-          dimensions.present? ? dimensions.with_indifferent_access : {}
-        end
-
-        def parse_metrics(metrics)
-          raise ValidationError, 'Must specify at least one metric' unless metrics.present?
-          metrics.with_indifferent_access
-        end
-      end
-    end
-  end
-end

data/lib/druid/writer/tranquilizer/dimensions.rb

@@ -1,16 +0,0 @@
-module Druid
-  module Writer
-    module Tranquilizer
-      class Dimensions
-        class << self
-          java_import com.metamx.tranquility.druid.DruidDimensions
-
-          def build(dimensions)
-            dimensions_list = ImmutableList.of(*dimensions.keys)
-            DruidDimensions.specific(dimensions_list)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/druid/writer/tranquilizer/druid_beam_config.rb

@@ -1,40 +0,0 @@
-# Defaults based on https://github.com/druid-io/tranquility/blob/master/core/src/main/scala/com/metamx/tranquility/druid/DruidBeamConfig.scala
-module Druid
-  module Writer
-    module Tranquilizer
-      module DruidBeamConfig
-        class << self
-          java_import com.metamx.tranquility.druid.DruidBeamConfig
-          java_import com.metamx.tranquility.druid.OverlordLocator
-          java_import com.metamx.tranquility.druid.TaskLocator
-
-          def build(randomize_task_id)
-            firehoseGracePeriod = org.joda.time.Period.new('PT5M')
-            firehoseQuietPeriod = org.joda.time.Period.new('PT1M')
-            firehoseRetryPeriod = org.joda.time.Period.new('PT1M')
-            firehoseChunkSize = 1000
-            randomizeTaskId = randomize_task_id
-            indexRetryPeriod = org.joda.time.Period.new('PT1M')
-            firehoseBufferSize = 100000
-            overlordLocator = OverlordLocator.Curator
-            taskLocator = TaskLocator.Curator
-            overlordPollPeriod = org.joda.time.Period.new('PT20S')
-
-            DruidBeamConfig.new(
-              firehoseGracePeriod,
-              firehoseQuietPeriod,
-              firehoseRetryPeriod,
-              firehoseChunkSize,
-              randomizeTaskId,
-              indexRetryPeriod,
-              firehoseBufferSize,
-              overlordLocator,
-              taskLocator,
-              overlordPollPeriod
-            )
-          end
-        end
-      end
-    end
-  end
-end

data/lib/druid/writer/tranquilizer/druid_beams.rb

@@ -1,12 +0,0 @@
-module Druid
-  module Writer
-    module Tranquilizer
-      java_import com.metamx.tranquility.druid.DruidBeams
-
-      class DruidBeams
-        # Ensures the correct override is used.
-        java_alias :timestamper_builder, :builder, [com.metamx.tranquility.typeclass.Timestamper.java_class]
-      end
-    end
-  end
-end

data/lib/druid/writer/tranquilizer/event_listener.rb

@@ -1,18 +0,0 @@
-module Druid
-  module Writer
-    module Tranquilizer
-      class EventListener
-        include Logging
-        include_package com.twitter.util.FutureEventListener
-
-        def onSuccess(data)
-          # logger.debug data.to_s
-        end
-
-        def onFailure(error)
-          logger.warn error.to_s
-        end
-      end
-    end
-  end
-end

data/lib/druid/writer/tranquilizer/future.rb

@@ -1,39 +0,0 @@
-module Druid
-  module Writer
-    module Tranquilizer
-      class Future
-        AWAIT = com.twitter.util.Awaitable::CanAwait
-        WAIT_TIME = 20
-
-        attr_reader :future
-        delegate :isDefined, to: :future
-
-        def initialize(future)
-          @future = future
-        end
-
-        def failure?(wait_time  = WAIT_TIME)
-          begin
-            future.ready(build_duration(wait_time), AWAIT).isThrow
-          rescue Java::ComTwitterUtil::TimeoutException => e
-            raise Druid::ConnectionError, 'Future timed out.'
-          end
-        end
-
-        def success?(wait_time = WAIT_TIME)
-          begin
-            future.ready(build_duration(wait_time), AWAIT).isReturn
-          rescue Java::ComTwitterUtil::TimeoutException => e
-            raise Druid::ConnectionError, 'Future timed out.'
-          end
-        end
-
-        private
-
-        def build_duration(duration)
-          com.twitter.util.Duration.fromSeconds(duration)
-        end
-      end
-    end
-  end
-end