efigence-influxdb 0.1.0

data/lib/influxdb/query/continuous_query.rb

@@ -0,0 +1,36 @@
+module InfluxDB
+  module Query
+    module ContinuousQuery # :nodoc:
+      def continuous_queries(database)
+        resp = execute("SHOW CONTINUOUS QUERIES", parse: true)
+        fetch_series(resp).select {|v| v['name'] == database}
+                          .fetch(0, {})
+                          .fetch('values', [])
+                          .map {|v| {'name' => v.first, 'query' => v.last}}
+      end
+      # # @example
+      # #
+      # # db.create_continuous_query(
+      # #   "select mean(sys) as sys, mean(usr) as usr from cpu group by time(15m)",
+      # #   "cpu.15m",
+      # # )
+      # #
+      # # NOTE: Only cluster admin can call this
+      # def create_continuous_query(query, name)
+      #   query("#{query} into #{name}")
+      # end
+
+      # # NOTE: Only cluster admin can call this
+      # def list_continuous_queries
+      #   query("list continuous queries")
+      #     .fetch("continuous queries", [])
+      #     .map { |l| l["query"] }
+      # end
+
+      # # NOTE: Only cluster admin can call this
+      # def delete_continuous_query(id)
+      #   query("drop continuous query #{id}")
+      # end
+    end
+  end
+end

data/lib/influxdb/query/core.rb

@@ -0,0 +1,109 @@
+module InfluxDB
+  module Query # :nodoc: all
+    # rubocop:disable Metrics/MethodLength
+    # rubocop:disable Metrics/AbcSize
+    module Core
+
+      def ping
+        get "/ping"
+      end
+
+      def query(query, opts = {})
+        precision   = opts.fetch(:precision, config.time_precision)
+        denormalize = opts.fetch(:denormalize, config.denormalize)
+
+        url = full_url("/query", q: query, db: config.database, precision: precision)
+        resp = get(url, parse: true)
+        series = fetch_series(resp)
+
+        if block_given?
+          series.each { |s| yield s['name'], s['tags'], denormalize ? denormalize_series(s) : raw_values(s) }
+        else
+          denormalize ? list_series(series) : series
+        end
+      end
+
+      # Example:
+      # write_points([
+      #   {
+      #     series: 'cpu',
+      #     tags: { host: 'server_nl', regios: 'us' },
+      #     values: {internal: 5, external: 6},
+      #     timestamp: 1422568543702900257
+      #   },
+      #   {
+      #     series: 'gpu',
+      #     values: {value: 0.9999},
+      #   }
+      # ])
+      def write_points(data, precision = nil)
+        data = data.is_a?(Array) ? data : [data]
+        payload = generate_payload(data)
+        writer.write(payload, precision)
+      end
+
+      # Example:
+      # write_point('cpu', tags: {region: 'us'}, values: {internal: 60})
+      def write_point(series, data, precision = nil)
+        data.merge!(series: series)
+        write_points(data, precision)
+      end
+
+      def write(data, precision)
+        precision ||= config.time_precision
+        url = full_url("/write", db: config.database, precision: precision)
+        post(url, data)
+      end
+
+      private
+
+      def list_series(series)
+        series.map do |s|
+          {
+            'name' => s['name'],
+            'tags' => s['tags'],
+            'values' => denormalize_series(s)
+          }
+        end
+      end
+
+      def fetch_series(response)
+        response.fetch('results', [])
+                .fetch(0, {})
+                .fetch('series', [])
+      end
+
+      def generate_payload(data)
+        data.map do |point|
+          InfluxDB::PointValue.new(point).dump
+        end.join("\n")
+      end
+
+      def execute(query, options={})
+        url = full_url("/query", q: query)
+        get(url, options)
+      end
+
+      def denormalize_series(series)
+        series["values"].map do |values|
+          Hash[series["columns"].zip(values)]
+        end
+      end
+
+      def raw_values(series)
+        series.select { |k,_| %w(columns values).include?(k) }
+      end
+
+      def full_url(path, params = {})
+        unless basic_auth?
+          params[:u] = config.username
+          params[:p] = config.password
+        end
+
+        query = params.map { |k, v| [CGI.escape(k.to_s), "=", CGI.escape(v.to_s)].join }.join("&")
+
+        URI::Generic.build(path: path, query: query).to_s
+      end
+    end
+  end
+end

data/lib/influxdb/query/database.rb

@@ -0,0 +1,21 @@
+module InfluxDB
+  module Query
+    module Database # :nodoc:
+      def create_database(name)
+        execute("CREATE DATABASE #{name}")
+      end
+
+      def delete_database(name)
+        execute("DROP DATABASE #{name}")
+      end
+
+      def list_databases
+        resp = execute("SHOW DATABASES", parse: true)
+        fetch_series(resp).fetch(0, {})
+                          .fetch('values', [])
+                          .flatten
+                          .map {|v| { 'name' => v }}
+      end
+    end
+  end
+end

data/lib/influxdb/query/series.rb

@@ -0,0 +1,13 @@
+module InfluxDB
+  module Query
+    module Series # :nodoc:
+      # def delete_series(series)
+      #   delete full_url("/db/#{config.database}/series/#{series}")
+      # end
+
+      # def list_series
+      #   query('list series').fetch('list_series_result').map { |l| l['name'] }
+      # end
+    end
+  end
+end

data/lib/influxdb/query/shard.rb

@@ -0,0 +1,14 @@
+module InfluxDB
+  module Query
+    module Shard # :nodoc:
+      # def list_shards
+      #   get full_url("/cluster/shards")
+      # end
+
+      # def delete_shard(shard_id, server_ids)
+      #   data = JSON.generate("serverIds" => server_ids)
+      #   delete full_url("/cluster/shards/#{shard_id}"), data
+      # end
+    end
+  end
+end

data/lib/influxdb/query/shard_space.rb

@@ -0,0 +1,60 @@
+module InfluxDB
+  module Query
+    module ShardSpace # :nodoc:
+      # def configure_database(database_name, options = {})
+      #   url  = full_url("/cluster/database_configs/#{database_name}")
+      #   data = JSON.generate(default_database_configuration.merge(options))
+
+      #   post(url, data)
+      # end
+
+      # def list_shard_spaces
+      #   get full_url("/cluster/shard_spaces")
+      # end
+
+      # def shard_space_info(database_name, shard_space_name)
+      #   list_shard_spaces.find do |shard_space|
+      #     shard_space["database"] == database_name &&
+      #       shard_space["name"] == shard_space_name
+      #   end
+      # end
+
+      # def create_shard_space(database_name, options = {})
+      #   url  = full_url("/cluster/shard_spaces/#{database_name}")
+      #   data = JSON.generate(default_shard_space_options.merge(options))
+
+      #   post(url, data)
+      # end
+
+      # def delete_shard_space(database_name, shard_space_name)
+      #   delete full_url("/cluster/shard_spaces/#{database_name}/#{shard_space_name}")
+      # end
+
+      # ## Get the shard space first, so the user doesn't have to specify the existing options
+      # def update_shard_space(database_name, shard_space_name, options)
+      #   shard_space_options = get_shard_space(database_name, shard_space_name)
+      #   shard_space_options.delete("database")
+
+      #   url  = full_url("/cluster/shard_spaces/#{database_name}/#{shard_space_name}")
+      #   data = JSON.generate(shard_space_options.merge(options))
+
+      #   post(url, data)
+      # end
+
+      # def default_shard_space_options
+      #   {
+      #     "name"              => "default",
+      #     "regEx"             => "/.*/",
+      #     "retentionPolicy"   => "inf",
+      #     "shardDuration"     => "7d",
+      #     "replicationFactor" => 1,
+      #     "split"             => 1
+      #   }
+      # end
+
+      # def default_database_configuration
+      #   { spaces:  [default_shard_space_options] }
+      # end
+    end
+  end
+end

data/lib/influxdb/query/user.rb

@@ -0,0 +1,38 @@
+module InfluxDB
+  module Query
+    module User # :nodoc:
+      # create_database_user('testdb', 'user', 'pass') => grants all privileges by default
+      # create_database_user('testdb', 'user', 'pass', :permissions => :read) => use [:read|:write|:all]
+      def create_database_user(database, username, password, options={})
+        permissions = options.fetch(:permissions, :all)
+        execute("CREATE user #{username} WITH PASSWORD '#{password}'; GRANT #{permissions.to_s.upcase} ON #{database} TO #{username}")
+      end
+
+      def update_user_password(username, password)
+        execute("SET PASSWORD FOR #{username} = '#{password}'")
+      end
+
+      # permission => [:read|:write|:all]
+      def grant_user_privileges(username, database, permission)
+        execute("GRANT #{permission.to_s.upcase} ON #{database} TO #{username}")
+      end
+
+      # permission => [:read|:write|:all]
+      def revoke_user_privileges(username, database, permission)
+        execute("REVOKE #{permission.to_s.upcase} ON #{database} FROM #{username}")
+      end
+
+      def delete_user(username)
+        execute("DROP USER #{username}")
+      end
+
+      # => [{"username"=>"usr", "admin"=>true}, {"username"=>"justauser", "admin"=>false}]
+      def list_users
+        resp = execute("SHOW USERS", parse: true)
+        fetch_series(resp).fetch(0, {})
+                          .fetch('values', [])
+                          .map {|v| {'username' => v.first, 'admin' => v.last}}
+      end
+    end
+  end
+end

data/lib/influxdb/version.rb

@@ -0,0 +1,3 @@
+module InfluxDB # :nodoc:
+  VERSION = "0.1.0"
+end

data/lib/influxdb/writer/async.rb

@@ -0,0 +1,115 @@
+require 'thread'
+require "net/http"
+require "uri"
+
+module InfluxDB
+  module Writer # :nodoc: all
+    class Async
+      attr_reader :config, :client
+
+      def initialize(client, config)
+        @client = client
+        @config = config
+      end
+
+      def write(data, precision = nil)
+        data = data.is_a?(Array) ? data : [data]
+        data.map { |p| worker.push(p) }
+      end
+
+      WORKER_MUTEX = Mutex.new
+      def worker
+        return @worker if @worker
+        WORKER_MUTEX.synchronize do
+          # this return is necessary because the previous mutex holder
+          # might have already assigned the @worker
+          return @worker if @worker
+          @worker = Worker.new(client, config)
+        end
+      end
+
+      class Worker
+        attr_reader :client, :queue, :threads
+
+        include InfluxDB::Logging
+
+        MAX_POST_POINTS = 1000
+        MAX_QUEUE_SIZE = 10_000
+        NUM_WORKER_THREADS = 3
+        SLEEP_INTERVAL = 5
+
+        def initialize(client, config)
+          @client = client
+          config = config.is_a?(Hash) ? config : {}
+          @queue = InfluxDB::MaxQueue.new config.fetch(:max_queue, MAX_QUEUE_SIZE)
+
+          spawn_threads!
+
+          at_exit do
+            log :debug, "Thread exiting, flushing queue."
+            check_background_queue until queue.empty?
+          end
+        end
+
+        def push(payload)
+          queue.push(payload)
+        end
+
+        def current_threads
+          Thread.list.select { |t| t[:influxdb] == object_id }
+        end
+
+        def current_thread_count
+          Thread.list.count { |t| t[:influxdb] == object_id }
+        end
+
+        # rubocop:disable Metrics/CyclomaticComplexity
+        # rubocop:disable Metrics/MethodLength
+        # rubocop:disable Metrics/AbcSize
+
+        def spawn_threads!
+          @threads = []
+          NUM_WORKER_THREADS.times do |thread_num|
+            log :debug, "Spawning background worker thread #{thread_num}."
+
+            @threads << Thread.new do
+              Thread.current[:influxdb] = object_id
+
+              until client.stopped?
+                check_background_queue(thread_num)
+                sleep rand(SLEEP_INTERVAL)
+              end
+
+              log :debug, "Exit background worker thread #{thread_num}."
+            end
+          end
+        end
+
+        def check_background_queue(thread_num = 0)
+          log :debug,
+              "Checking background queue on thread #{thread_num} (#{current_thread_count} active)"
+
+          loop do
+            data = []
+
+            while data.size < MAX_POST_POINTS && !queue.empty?
+              p = queue.pop(true) rescue next
+              data.push p
+            end
+
+            return if data.empty?
+
+            begin
+              log :debug, "Found data in the queue! (#{data.length} points)"
+              client.write(data)
+            rescue => e
+              puts "Cannot write data: #{e.inspect}"
+            end
+
+            break if queue.length > MAX_POST_POINTS
+          end
+        end
+      end
+    end
+  end
+end

data/lib/influxdb/writer/udp.rb

@@ -0,0 +1,21 @@
+module InfluxDB
+  module Writer
+    # Writes data to InfluxDB through UDP
+    class UDP
+      attr_accessor :socket
+      attr_reader :host, :port
+      def initialize(client, config)
+        @client = client
+        config = config.is_a?(Hash) ? config : {}
+        @host = config.fetch(:host, 'localhost')
+        @port = config.fetch(:port, 4444)
+        self.socket = UDPSocket.new
+        socket.connect(host, port)
+      end
+
+      def write(payload, precision = nil)
+        socket.send(payload, 0)
+      end
+    end
+  end
+end

data/spec/influxdb/cases/async_client_spec.rb

@@ -0,0 +1,33 @@
+require "spec_helper"
+require "timeout"
+
+describe InfluxDB::Client do
+  let(:subject) { described_class.new(async: true) }
+
+  let(:worker_klass) { InfluxDB::Writer::Async::Worker }
+
+  specify { expect(subject.writer).to be_a(InfluxDB::Writer::Async) }
+
+  describe "#write_point" do
+    let(:payload) { "responses,region=eu value=5" }
+
+    it "sends writes to client" do
+      # exact times can be 2 or 3 (because we have 3 worker threads),
+      # but cannot be less than 2 due to MAX_POST_POINTS limit
+      expect(subject).to(receive(:write)).at_least(2).times
+
+      (worker_klass::MAX_POST_POINTS + 100).times do
+        subject.write_point('a', {})
+      end
+
+      Timeout.timeout(2 * worker_klass::SLEEP_INTERVAL) do
+        subject.stop!
+        # ensure threads exit
+        subject.writer.worker.threads.each(&:join)
+
+        # flush queue (we cannot test `at_exit`)
+        subject.writer.worker.check_background_queue
+      end
+    end
+  end
+end