jruby-druid 1.0.0.pre.rc2

data/lib/druid/node/coordinator.rb

@@ -0,0 +1,108 @@
+module Druid
+  module Node
+    class Coordinator
+      DATASOURCES_PATH = '/druid/coordinator/v1/datasources/'.freeze
+
+      attr_reader :connection
+      def initialize(config)
+        @connection = Druid::Connection.new(config.coordinator_uri)
+      end
+
+      def datasource_info(datasource_name)
+        response = connection.get(DATASOURCES_PATH + datasource_name.to_s, full: true)
+        raise ConnectionError, 'Unable to retrieve datasource information.' unless response.code.to_i == 200
+        JSON.parse(response.body)
+      end
+
+      def disable_datasource(datasource_name)
+        # response = connection.delete(DATASOURCES_PATH + datasource_name.to_s)
+        # raise ConnectionError, 'Unable to disable datasource' unless response.code.to_i == 200
+        # return true if response.code.to_i == 200
+
+        # This is a workaround for https://github.com/druid-io/druid/issues/3154
+        disable_segments(datasource_name)
+        bounded_wait_for_segments_disable(datasource_name)
+        true
+      end
+
+      # TODO: This should either be private or moved to datasource
+      def datasource_enabled?(datasource_name)
+        list_datasources.include? datasource_name
+      end
+
+      # TODO: This should either be private or moved to datasource
+      def datasource_has_segments?(datasource_name)
+        list_segments(datasource_name).any?
+      end
+
+      def disable_segment(datasource_name, segment)
+        response = connection.delete(DATASOURCES_PATH + datasource_name + '/segments/' + segment)
+        raise ConnectionError, "Unable to disable #{segment}" unless response.code.to_i == 200
+        true
+      end
+
+      # TODO: This should either be private or moved to datasource
+      def disable_segments(datasource_name)
+        segments = list_segments(datasource_name)
+        segments.each{ |segment| disable_segment(datasource_name, segment) }
+      end
+
+      def issue_kill_task(datasource_name, interval)
+        response = connection.delete(DATASOURCES_PATH + datasource_name + '/intervals/' + interval)
+        raise ConnectionError, 'Unable to issue kill task.' unless response.code.to_i == 200
+        true
+      end
+
+      def list_datasources(url_params = {})
+        response = connection.get(DATASOURCES_PATH, url_params)
+        JSON.parse(response.body) if response.code.to_i == 200
+      end
+
+      def list_segments(datasource_name)
+        response = connection.get(DATASOURCES_PATH + datasource_name + '/segments', full: true)
+        case response.code.to_i
+        when 200
+          JSON.parse(response.body).map{ |segment| segment['identifier'] }
+        when 204
+          []
+        else
+          raise ConnectionError, "Unable to list segments for #{datasource_name}"
+        end
+      end
+
+      private
+
+      def bounded_wait_for_disable(datasource_name)
+        condition = datasource_enabled?(datasource_name)
+        attempts = 0
+        max = 10
+
+        while(condition) do
+          attempts += 1
+          sleep 1
+          condition = datasource_enabled?(datasource_name)
+          break if attempts >= max
+        end
+
+        raise ClientError, 'Datasource should be disabled, but is still enabled.' unless condition
+        true
+      end
+
+      def bounded_wait_for_segments_disable(datasource_name)
+        condition = datasource_has_segments?(datasource_name)
+        attempts = 0
+        max = 60
+
+        while(condition) do
+          attempts += 1
+          sleep 1
+          condition = datasource_has_segments?(datasource_name)
+          break if attempts >= max
+        end
+
+        raise ClientError, 'Segments should be disabled, but are still enabled.' if condition
+        true
+      end
+    end
+  end
+end

data/lib/druid/node/overlord.rb

@@ -0,0 +1,51 @@
+module Druid
+  module Node
+    class Overlord
+      INDEXER_PATH = '/druid/indexer/v1/'.freeze
+      RUNNING_TASKS_PATH = (INDEXER_PATH + 'runningTasks').freeze
+      TASK_PATH = INDEXER_PATH + 'task/'
+
+      attr_reader :connection
+      def initialize(config)
+        @connection = Druid::Connection.new(config.overlord_uri)
+      end
+
+      def running_tasks(datasource_name = nil)
+        response = connection.get(RUNNING_TASKS_PATH)
+        raise ConnectionError, 'Could not retrieve running tasks' unless response.code.to_i == 200
+        tasks = JSON.parse(response.body).map{|task| task['id']}
+        tasks.select!{ |task| task.include? datasource_name } if datasource_name
+        tasks ? tasks : []
+      end
+
+      def shutdown_task(task)
+        response = connection.post(TASK_PATH + task + '/shutdown')
+        raise ConnectionError, 'Unable to shutdown task' unless response.code.to_i == 200
+        bounded_wait_for_shutdown(task)
+      end
+
+      def shutdown_tasks(datasource_name = nil)
+        tasks = running_tasks(datasource_name)
+        tasks.each{|task| shutdown_task(task)}
+      end
+
+      private
+
+      def bounded_wait_for_shutdown(task)
+        condition = !(running_tasks.include? task)
+        attempts = 0
+        max = 10
+
+        until(condition) do
+          attempts += 1
+          sleep 1
+          condition = !(running_tasks.include? task)
+          break if attempts >= max
+        end
+
+        raise ClientError, 'Task did not shutdown.' unless condition
+        true
+      end
+    end
+  end
+end

data/lib/druid/query/core.rb

@@ -0,0 +1,8 @@
+module Druid
+  module Query
+    module Core
+      delegate :query, to: :broker
+      delegate :write_point, to: :writer
+    end
+  end
+end

data/lib/druid/query/datasource.rb

@@ -0,0 +1,35 @@
+module Druid
+  module Query
+    module Datasource
+      java_import org.apache.zookeeper.ZKUtil
+
+      ZOOKEEPER_BEAMS_PATH = '/tranquility/beams/druid:overlord'.freeze
+
+      delegate :datasource_enabled?,
+               :datasource_info,
+               :disable_datasource,
+               :list_datasources,
+               to: :coordinator
+
+      def delete_datasource(datasource_name)
+        shutdown_tasks(datasource_name)
+        datasource_enabled?(datasource_name) ? disable_datasource(datasource_name) : true
+        writer.remove_tranquilizer_for_datasource(datasource_name)
+        delete_zookeeper_nodes(datasource_name) if config.strong_delete
+      end
+
+      def delete_datasources
+        list_datasources.each{ |datasource_name| delete_datasource(datasource_name) }
+      end
+
+      private
+
+      def delete_zookeeper_nodes(datasource_name)
+        curator = Druid::Writer::Tranquilizer::Curator.build(config)
+        curator.start
+        zk = curator.getZookeeperClient.getZooKeeper
+        ZKUtil.deleteRecursive(zk, ZOOKEEPER_BEAMS_PATH + "/#{datasource_name}")
+      end
+    end
+  end
+end

data/lib/druid/query/task.rb

@@ -0,0 +1,7 @@
+module Druid
+  module Query
+    module Task
+      delegate :shutdown_tasks, to: :overlord
+    end
+  end
+end

data/lib/druid/version.rb

@@ -0,0 +1,3 @@
+module JrubyDruid
+  VERSION = '1.0.0-rc2'
+end

data/lib/druid/writer/base.rb

@@ -0,0 +1,59 @@
+module Druid
+  module Writer
+    class Base
+      attr_accessor :tranquilizers
+      attr_reader :config
+
+      def initialize(config)
+        @config = config
+        @tranquilizers = []
+      end
+
+      def remove_tranquilizer_for_datasource(datasource)
+        tranquilizer = tranquilizer_for_datasource(datasource)
+        remove_tranquilizer(tranquilizer) if tranquilizer
+      end
+
+      def write_point(datasource, datapoint)
+        datapoint = Druid::Writer::Tranquilizer::Datapoint.new(datapoint)
+        sender = get_tranquilizer(datasource, datapoint)
+        sender.send(datapoint)
+      end
+
+      private
+
+      def build_tranquilizer(datasource, datapoint)
+        Druid::Writer::Tranquilizer::Base.new({config: config, datasource: datasource, datapoint: datapoint})
+      end
+
+      def get_tranquilizer(datasource, datapoint)
+        tranquilizer = tranquilizer_for_datasource(datasource)
+
+        unless has_current_schema?(tranquilizer, datapoint)
+          remove_tranquilizer(tranquilizer) if tranquilizer
+          tranquilizer = build_tranquilizer(datasource, datapoint)
+          tranquilizers << tranquilizer
+        end
+
+        tranquilizer
+      end
+
+      def has_current_schema?(tranquilizer, datapoint)
+        return false unless tranquilizer
+        dimensions = tranquilizer.rollup.dimensions.specMap["dimensions"].to_a
+        aggregators = tranquilizer.rollup.aggregators
+        metrics = Java::ScalaCollection::JavaConverters.seqAsJavaListConverter(aggregators).asJava.to_a.map{ |metric| metric.getFieldName }
+        dimensions == datapoint.dimensions.keys && metrics == datapoint.metrics.keys
+      end
+
+      def remove_tranquilizer(tranquilizer)
+        tranquilizers.delete(tranquilizer)
+        tranquilizer.stop
+      end
+
+      def tranquilizer_for_datasource(datasource)
+        tranquilizers.detect{ |t| t.datasource == datasource }
+      end
+    end
+  end
+end

data/lib/druid/writer/tranquilizer.rb

@@ -0,0 +1,48 @@
+module Druid
+  module Writer
+    module Tranquilizer
+      class << self
+        def ch
+          Java::Ch
+        end
+
+        def io
+          Java::Io
+        end
+      end
+      
+      java_import com.google.common.collect.ImmutableList
+      java_import com.google.common.collect.ImmutableMap
+      java_import ch.qos.logback.classic.Level
+      java_import ch.qos.logback.classic.encoder.PatternLayoutEncoder
+      java_import ch.qos.logback.core.FileAppender
+      java_import com.metamx.tranquility.beam.ClusteredBeamTuning
+      java_import io.druid.query.aggregation.LongSumAggregatorFactory
+      java_import io.druid.granularity.QueryGranularity
+      java_import io.druid.data.input.impl.TimestampSpec
+      java_import org.apache.curator.framework.CuratorFrameworkFactory
+      java_import org.slf4j.LoggerFactory
+      java_import org.slf4j.Logger
+      java_import Java::ScalaCollection::JavaConverters
+
+      logger = Druid::Writer::Tranquilizer::LoggerFactory.getLogger(Druid::Writer::Tranquilizer::Logger.ROOT_LOGGER_NAME)
+      logger.detachAndStopAllAppenders
+
+      appender = Druid::Writer::Tranquilizer::FileAppender.new
+      context = logger.getLoggerContext
+      encoder = Druid::Writer::Tranquilizer::PatternLayoutEncoder.new
+
+      encoder.setPattern("%date %level [%thread] %logger{10} [%file:%line] %msg%n")
+      encoder.setContext(context)
+      encoder.start
+
+      appender.setFile('jruby-druid.log')
+      appender.setEncoder(encoder)
+      appender.setContext(context)
+      appender.start
+
+      logger.addAppender(appender)
+      logger.setLevel(Druid::Writer::Tranquilizer::Level::TRACE)
+    end
+  end
+end

data/lib/druid/writer/tranquilizer/aggregators.rb

@@ -0,0 +1,15 @@
+module Druid
+  module Writer
+    module Tranquilizer
+      class Aggregators
+        class << self
+          def build(metrics)
+            aggs = []
+            metrics.keys.each{ |metric_name| aggs << LongSumAggregatorFactory.new(metric_name, metric_name) }
+            ImmutableList.of(*aggs)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/druid/writer/tranquilizer/base.rb

@@ -0,0 +1,64 @@
+# Based on Java Example below
+# https://github.com/gianm/tranquility-example/blob/master/src/main/java/io/imply/tranquility/JavaExample.java
+
+module Druid
+  module Writer
+    module Tranquilizer
+      class Base
+        attr_reader :config, :curator, :datasource, :rollup, :service, :tuning
+
+        def initialize(params)
+          @config = params[:config]
+          @datasource = params[:datasource].to_s
+          @rollup = Druid::Writer::Tranquilizer::Rollup.build(config, params[:datapoint])
+          @tuning = Druid::Writer::Tranquilizer::Tuning.build(config)
+          @curator = Druid::Writer::Tranquilizer::Curator.build(config)
+          @service = create_service
+
+          start
+        end
+
+        def send(datapoint)
+          service.send(argument_map(datapoint)).addEventListener(EventListener.new)
+        end
+
+        def start
+          curator.start
+          service.start
+        end
+
+        def stop
+          service.flush
+          service.stop
+          curator.close
+        end
+
+        private
+
+        def argument_map(datapoint)
+          args = datapoint.timestamp.
+            merge(datapoint.dimensions).
+            merge(datapoint.metrics)
+
+          ImmutableMap.builder.putAll(args).build
+        end
+
+        def create_service
+          timestamper = Timestamper.new
+          timestamp_spec = TimestampSpec.new("timestamp", "auto", nil)
+
+          builder = DruidBeams.
+            timestamper_builder(timestamper).
+            curator(curator).
+            discoveryPath(config.discovery_path).
+            location(com.metamx.tranquility.druid.DruidLocation.create(config.index_service, datasource)).
+            timestampSpec(timestamp_spec).
+            rollup(rollup).
+            tuning(tuning)
+          builder = builder.druidBeamConfig(DruidBeamConfig.build(true)) if config.strong_delete
+          builder.buildTranquilizer
+        end
+      end
+    end
+  end
+end

data/lib/druid/writer/tranquilizer/curator.rb

@@ -0,0 +1,18 @@
+module Druid
+  module Writer
+    module Tranquilizer
+      class Curator
+        class << self
+          def build(config)
+            bounded_retry = org.apache.curator.retry.BoundedExponentialBackoffRetry.new(100, 30000, 29)
+            CuratorFrameworkFactory.
+              builder.
+              connectString(config.curator_uri).
+              retryPolicy(bounded_retry).
+              build
+          end
+        end
+      end
+    end
+  end
+end

data/lib/druid/writer/tranquilizer/datapoint.rb

@@ -0,0 +1,24 @@
+module Druid
+  module Writer
+    module Tranquilizer
+      class Datapoint
+        TIMESTAMP_LABEL = 'timestamp'.freeze
+
+        attr_reader :dimensions, :timestamp, :metrics
+
+        def initialize(datapoint)
+          @timestamp = build_time(datapoint[:timestamp])
+          @dimensions = datapoint[:dimensions].with_indifferent_access
+          @metrics = datapoint[:metrics].with_indifferent_access
+        end
+
+        private
+
+        def build_time(time)
+          time = Time.now.utc unless time
+          Hash[TIMESTAMP_LABEL, time.iso8601]
+        end
+      end
+    end
+  end
+end