influxdb 0.1.9 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/.rubocop.yml +41 -0
  4. data/.travis.yml +3 -2
  5. data/Gemfile +7 -1
  6. data/README.md +218 -102
  7. data/Rakefile +2 -6
  8. data/lib/influxdb.rb +15 -5
  9. data/lib/influxdb/client.rb +38 -433
  10. data/lib/influxdb/client/http.rb +123 -0
  11. data/lib/influxdb/config.rb +66 -0
  12. data/lib/influxdb/errors.rb +8 -2
  13. data/lib/influxdb/{logger.rb → logging.rb} +6 -5
  14. data/lib/influxdb/max_queue.rb +2 -1
  15. data/lib/influxdb/point_value.rb +27 -25
  16. data/lib/influxdb/query/cluster.rb +17 -0
  17. data/lib/influxdb/query/continuous_query.rb +22 -0
  18. data/lib/influxdb/query/core.rb +110 -0
  19. data/lib/influxdb/query/database.rb +21 -0
  20. data/lib/influxdb/query/retention_policy.rb +26 -0
  21. data/lib/influxdb/query/user.rb +41 -0
  22. data/lib/influxdb/version.rb +2 -2
  23. data/lib/influxdb/writer/async.rb +115 -0
  24. data/lib/influxdb/writer/udp.rb +21 -0
  25. data/spec/influxdb/cases/async_client_spec.rb +33 -0
  26. data/spec/influxdb/cases/query_cluster_spec.rb +65 -0
  27. data/spec/influxdb/cases/query_continuous_query_spec.rb +82 -0
  28. data/spec/influxdb/cases/query_core.rb +34 -0
  29. data/spec/influxdb/cases/query_database_spec.rb +58 -0
  30. data/spec/influxdb/cases/query_retention_policy_spec.rb +84 -0
  31. data/spec/influxdb/cases/query_series_spec.rb +50 -0
  32. data/spec/influxdb/cases/query_shard_space_spec.rb +105 -0
  33. data/spec/influxdb/cases/query_shard_spec.rb +43 -0
  34. data/spec/influxdb/cases/query_user_spec.rb +127 -0
  35. data/spec/influxdb/cases/querying_spec.rb +149 -0
  36. data/spec/influxdb/cases/retry_requests_spec.rb +102 -0
  37. data/spec/influxdb/cases/udp_client_spec.rb +21 -0
  38. data/spec/influxdb/cases/write_points_spec.rb +140 -0
  39. data/spec/influxdb/client_spec.rb +37 -810
  40. data/spec/influxdb/config_spec.rb +118 -0
  41. data/spec/influxdb/{logger_spec.rb → logging_spec.rb} +4 -8
  42. data/spec/influxdb/max_queue_spec.rb +29 -0
  43. data/spec/influxdb/point_value_spec.rb +81 -14
  44. data/spec/influxdb/worker_spec.rb +8 -11
  45. data/spec/spec_helper.rb +7 -10
  46. metadata +65 -30
  47. data/lib/influxdb/udp_client.rb +0 -16
  48. data/lib/influxdb/worker.rb +0 -80
  49. data/spec/influxdb/udp_client_spec.rb +0 -33
  50. data/spec/influxdb_spec.rb +0 -4
  51. data/spec/max_queue_spec.rb +0 -32
@@ -0,0 +1,21 @@
1
+ module InfluxDB
2
+ module Query
3
+ module Database # :nodoc:
4
+ def create_database(name)
5
+ execute("CREATE DATABASE #{name}")
6
+ end
7
+
8
+ def delete_database(name)
9
+ execute("DROP DATABASE #{name}")
10
+ end
11
+
12
+ def list_databases
13
+ resp = execute("SHOW DATABASES", parse: true)
14
+ fetch_series(resp).fetch(0, {})
15
+ .fetch('values', [])
16
+ .flatten
17
+ .map { |v| { 'name' => v } }
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,26 @@
1
+ module InfluxDB
2
+ module Query
3
+ module RetentionPolicy # :nodoc:
4
+ def create_retention_policy(name, database, duration, replication, default = false)
5
+ execute(
6
+ "CREATE RETENTION POLICY \"#{name}\" ON #{database} "\
7
+ "DURATION #{duration} REPLICATION #{replication}#{default ? ' DEFAULT' : ''}")
8
+ end
9
+
10
+ def list_retention_policies(database)
11
+ resp = execute("SHOW RETENTION POLICIES \"#{database}\"", parse: true)
12
+ data = fetch_series(resp).fetch(0)
13
+
14
+ data['values'].map do |policy|
15
+ policy.each.with_index.inject({}) do |hash, (value, index)|
16
+ hash.tap { |h| h[data['columns'][index]] = value }
17
+ end
18
+ end
19
+ end
20
+
21
+ def delete_retention_policy(name, database)
22
+ execute("DROP RETENTION POLICY \"#{name}\" ON #{database}")
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,41 @@
1
+ module InfluxDB
2
+ module Query
3
+ module User # :nodoc:
4
+ # create_database_user('testdb', 'user', 'pass') - grants all privileges by default
5
+ # create_database_user('testdb', 'user', 'pass', permissions: :read) - use [:read|:write|:all]
6
+ def create_database_user(database, username, password, options = {})
7
+ permissions = options.fetch(:permissions, :all)
8
+ execute(
9
+ "CREATE user #{username} WITH PASSWORD '#{password}'; "\
10
+ "GRANT #{permissions.to_s.upcase} ON #{database} TO #{username}"
11
+ )
12
+ end
13
+
14
+ def update_user_password(username, password)
15
+ execute("SET PASSWORD FOR #{username} = '#{password}'")
16
+ end
17
+
18
+ # permission => [:read|:write|:all]
19
+ def grant_user_privileges(username, database, permission)
20
+ execute("GRANT #{permission.to_s.upcase} ON #{database} TO #{username}")
21
+ end
22
+
23
+ # permission => [:read|:write|:all]
24
+ def revoke_user_privileges(username, database, permission)
25
+ execute("REVOKE #{permission.to_s.upcase} ON #{database} FROM #{username}")
26
+ end
27
+
28
+ def delete_user(username)
29
+ execute("DROP USER #{username}")
30
+ end
31
+
32
+ # => [{"username"=>"usr", "admin"=>true}, {"username"=>"justauser", "admin"=>false}]
33
+ def list_users
34
+ resp = execute("SHOW USERS", parse: true)
35
+ fetch_series(resp).fetch(0, {})
36
+ .fetch('values', [])
37
+ .map { |v| { 'username' => v.first, 'admin' => v.last } }
38
+ end
39
+ end
40
+ end
41
+ end
@@ -1,3 +1,3 @@
1
- module InfluxDB
2
- VERSION = "0.1.9"
1
+ module InfluxDB # :nodoc:
2
+ VERSION = "0.2.0"
3
3
  end
@@ -0,0 +1,115 @@
1
+ require 'thread'
2
+ require "net/http"
3
+ require "uri"
4
+
5
+ module InfluxDB
6
+ module Writer # :nodoc: all
7
+ class Async
8
+ attr_reader :config, :client
9
+
10
+ def initialize(client, config)
11
+ @client = client
12
+ @config = config
13
+ end
14
+
15
+ def write(data, _precision = nil)
16
+ data = data.is_a?(Array) ? data : [data]
17
+ data.map { |p| worker.push(p) }
18
+ end
19
+
20
+ WORKER_MUTEX = Mutex.new
21
+ def worker
22
+ return @worker if @worker
23
+ WORKER_MUTEX.synchronize do
24
+ # this return is necessary because the previous mutex holder
25
+ # might have already assigned the @worker
26
+ return @worker if @worker
27
+ @worker = Worker.new(client, config)
28
+ end
29
+ end
30
+
31
+ class Worker
32
+ attr_reader :client, :queue, :threads
33
+
34
+ include InfluxDB::Logging
35
+
36
+ MAX_POST_POINTS = 1000
37
+ MAX_QUEUE_SIZE = 10_000
38
+ NUM_WORKER_THREADS = 3
39
+ SLEEP_INTERVAL = 5
40
+
41
+ def initialize(client, config)
42
+ @client = client
43
+ config = config.is_a?(Hash) ? config : {}
44
+ @queue = InfluxDB::MaxQueue.new config.fetch(:max_queue, MAX_QUEUE_SIZE)
45
+
46
+ spawn_threads!
47
+
48
+ at_exit do
49
+ log :debug, "Thread exiting, flushing queue."
50
+ check_background_queue until queue.empty?
51
+ end
52
+ end
53
+
54
+ def push(payload)
55
+ queue.push(payload)
56
+ end
57
+
58
+ def current_threads
59
+ Thread.list.select { |t| t[:influxdb] == object_id }
60
+ end
61
+
62
+ def current_thread_count
63
+ Thread.list.count { |t| t[:influxdb] == object_id }
64
+ end
65
+
66
+ # rubocop:disable Metrics/CyclomaticComplexity
67
+ # rubocop:disable Metrics/MethodLength
68
+ # rubocop:disable Metrics/AbcSize
69
+
70
+ def spawn_threads!
71
+ @threads = []
72
+ NUM_WORKER_THREADS.times do |thread_num|
73
+ log :debug, "Spawning background worker thread #{thread_num}."
74
+
75
+ @threads << Thread.new do
76
+ Thread.current[:influxdb] = object_id
77
+
78
+ until client.stopped?
79
+ check_background_queue(thread_num)
80
+ sleep rand(SLEEP_INTERVAL)
81
+ end
82
+
83
+ log :debug, "Exit background worker thread #{thread_num}."
84
+ end
85
+ end
86
+ end
87
+
88
+ def check_background_queue(thread_num = 0)
89
+ log :debug,
90
+ "Checking background queue on thread #{thread_num} (#{current_thread_count} active)"
91
+
92
+ loop do
93
+ data = []
94
+
95
+ while data.size < MAX_POST_POINTS && !queue.empty?
96
+ p = queue.pop(true) rescue next
97
+ data.push p
98
+ end
99
+
100
+ return if data.empty?
101
+
102
+ begin
103
+ log :debug, "Found data in the queue! (#{data.length} points)"
104
+ client.write(data)
105
+ rescue => e
106
+ puts "Cannot write data: #{e.inspect}"
107
+ end
108
+
109
+ break if queue.length > MAX_POST_POINTS
110
+ end
111
+ end
112
+ end
113
+ end
114
+ end
115
+ end
@@ -0,0 +1,21 @@
1
+ module InfluxDB
2
+ module Writer
3
+ # Writes data to InfluxDB through UDP
4
+ class UDP
5
+ attr_accessor :socket
6
+ attr_reader :host, :port
7
+ def initialize(client, config)
8
+ @client = client
9
+ config = config.is_a?(Hash) ? config : {}
10
+ @host = config.fetch(:host, 'localhost')
11
+ @port = config.fetch(:port, 4444)
12
+ self.socket = UDPSocket.new
13
+ socket.connect(host, port)
14
+ end
15
+
16
+ def write(payload, _precision = nil)
17
+ socket.send(payload, 0)
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,33 @@
1
+ require "spec_helper"
2
+ require "timeout"
3
+
4
+ describe InfluxDB::Client do
5
+ let(:subject) { described_class.new(async: true) }
6
+
7
+ let(:worker_klass) { InfluxDB::Writer::Async::Worker }
8
+
9
+ specify { expect(subject.writer).to be_a(InfluxDB::Writer::Async) }
10
+
11
+ describe "#write_point" do
12
+ let(:payload) { "responses,region=eu value=5" }
13
+
14
+ it "sends writes to client" do
15
+ # exact times can be 2 or 3 (because we have 3 worker threads),
16
+ # but cannot be less than 2 due to MAX_POST_POINTS limit
17
+ expect(subject).to(receive(:write)).at_least(2).times
18
+
19
+ (worker_klass::MAX_POST_POINTS + 100).times do
20
+ subject.write_point('a', {})
21
+ end
22
+
23
+ Timeout.timeout(2 * worker_klass::SLEEP_INTERVAL) do
24
+ subject.stop!
25
+ # ensure threads exit
26
+ subject.writer.worker.threads.each(&:join)
27
+
28
+ # flush queue (we cannot test `at_exit`)
29
+ subject.writer.worker.check_background_queue
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,65 @@
1
+ require "spec_helper"
2
+ require "json"
3
+
4
+ describe InfluxDB::Client do
5
+ let(:subject) do
6
+ described_class.new(
7
+ "database",
8
+ {
9
+ host: "influxdb.test",
10
+ port: 9999,
11
+ username: "username",
12
+ password: "password",
13
+ time_precision: "s"
14
+ }.merge(args)
15
+ )
16
+ end
17
+
18
+ let(:args) { {} }
19
+
20
+ describe "#create_cluster_admin" do
21
+ let(:user) { 'adminadmin' }
22
+ let(:pass) { 'passpass' }
23
+ let(:query) { "CREATE USER #{user} WITH PASSWORD '#{pass}' WITH ALL PRIVILEGES" }
24
+
25
+ before do
26
+ stub_request(:get, "http://influxdb.test:9999/query").with(
27
+ query: { u: "username", p: "password", q: query }
28
+ )
29
+ end
30
+
31
+ it "should GET to create a new cluster admin" do
32
+ expect(subject.create_cluster_admin(user, pass)).to be_a(Net::HTTPOK)
33
+ end
34
+ end
35
+
36
+ describe "#list_cluster_admins" do
37
+ let(:response) { { "results" => [{ "series" => [{ "columns" => %w(user admin), "values" => [["dbadmin", true], ["foobar", false]] }] }] } }
38
+ let(:expected_result) { ["dbadmin"] }
39
+
40
+ before do
41
+ stub_request(:get, "http://influxdb.test:9999/query").with(
42
+ query: { u: "username", p: "password", q: "SHOW USERS" }
43
+ ).to_return(body: JSON.generate(response, status: 200))
44
+ end
45
+
46
+ it "should GET a list of cluster admins" do
47
+ expect(subject.list_cluster_admins).to eq(expected_result)
48
+ end
49
+ end
50
+
51
+ describe "#revoke_cluster_admin_privileges" do
52
+ let(:user) { 'useruser' }
53
+ let(:query) { "REVOKE ALL PRIVILEGES FROM #{user}" }
54
+
55
+ before do
56
+ stub_request(:get, "http://influxdb.test:9999/query").with(
57
+ query: { u: "username", p: "password", q: query }
58
+ )
59
+ end
60
+
61
+ it "should GET to revoke cluster admin privileges from a user" do
62
+ expect(subject.revoke_cluster_admin_privileges(user)).to be_a(Net::HTTPOK)
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,82 @@
1
+ require "spec_helper"
2
+ require "json"
3
+
4
+ describe InfluxDB::Client do
5
+ let(:subject) do
6
+ described_class.new(
7
+ "database",
8
+ {
9
+ host: "influxdb.test",
10
+ port: 9999,
11
+ username: "username",
12
+ password: "password",
13
+ time_precision: "s"
14
+ }.merge(args)
15
+ )
16
+ end
17
+
18
+ let(:args) { {} }
19
+
20
+ describe "#list_continuous_queries" do
21
+ let(:query) { "SHOW CONTINUOUS QUERIES" }
22
+ let(:database) { "testdb" }
23
+ let(:response) do
24
+ { "results" => [{ "series" => [{ "name" => "otherdb", "columns" => %w(name query),
25
+ "values" =>
26
+ [["clicks_per_hour", "CREATE CONTINUOUS QUERY clicks_per_hour ON otherdb BEGIN SELECT count(name) INTO \"otherdb\".\"default\".clicksCount_1h FROM \"otherdb\".\"default\".clicks GROUP BY time(1h) END"]] },
27
+ { "name" => "testdb", "columns" => %w(name query),
28
+ "values" =>
29
+ [["event_counts", "CREATE CONTINUOUS QUERY event_counts ON testdb BEGIN SELECT count(type) INTO \"testdb\".\"default\".typeCount_10m_byType FROM \"testdb\".\"default\".events GROUP BY time(10m), type END"]] }] }] }
30
+ end
31
+
32
+ let(:expected_result) do
33
+ [{ "name" => "event_counts", "query" => "CREATE CONTINUOUS QUERY event_counts ON testdb BEGIN SELECT count(type) INTO \"testdb\".\"default\".typeCount_10m_byType FROM \"testdb\".\"default\".events GROUP BY time(10m), type END" }]
34
+ end
35
+
36
+ before do
37
+ stub_request(:get, "http://influxdb.test:9999/query").with(
38
+ query: { u: "username", p: "password", q: query }
39
+ ).to_return(body: JSON.generate(response), status: 200)
40
+ end
41
+
42
+ it "should GET a list of continuous queries for specified db only" do
43
+ expect(subject.list_continuous_queries(database)).to eq(expected_result)
44
+ end
45
+ end
46
+
47
+ describe "#create_continuous_query" do
48
+ let(:name) { "event_counts_per_10m_by_type" }
49
+ let(:database) { "testdb" }
50
+ let(:query) do
51
+ "SELECT COUNT(type) INTO typeCount_10m_byType FROM events GROUP BY time(10m), type"
52
+ end
53
+ let(:clause) do
54
+ ["CREATE CONTINUOUS QUERY #{name} ON #{database} BEGIN", query, "END"].join("\n")
55
+ end
56
+
57
+ before do
58
+ stub_request(:get, "http://influxdb.test:9999/query").with(
59
+ query: { u: "username", p: "password", q: clause }
60
+ )
61
+ end
62
+
63
+ it "should GET to create a new continuous query" do
64
+ expect(subject.create_continuous_query(name, database, query)).to be_a(Net::HTTPOK)
65
+ end
66
+ end
67
+
68
+ describe "#delete_continuous_query" do
69
+ let(:name) { "event_counts_per_10m_by_type" }
70
+ let(:database) { "testdb" }
71
+ let(:query) { "DROP CONTINUOUS QUERY #{name} ON #{database}" }
72
+ before do
73
+ stub_request(:get, "http://influxdb.test:9999/query").with(
74
+ query: { u: "username", p: "password", q: query }
75
+ )
76
+ end
77
+
78
+ it "should GET to remove continuous query" do
79
+ expect(subject.delete_continuous_query(name, database)).to be_a(Net::HTTPOK)
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,34 @@
1
+ require 'spec_helper'
2
+ require 'json'
3
+
4
+ describe InfluxDB::Client do
5
+ let(:subject) do
6
+ described_class.new(
7
+ 'database',
8
+ {
9
+ host: 'influxdb.test',
10
+ port: 9999,
11
+ username: 'username',
12
+ password: 'password',
13
+ time_precision: 's'
14
+ }.merge(args)
15
+ )
16
+ end
17
+
18
+ let(:args) { {} }
19
+
20
+ describe '#query' do
21
+
22
+ it 'should handle responses with no values' do
23
+ # Some requests (such as trying to retrieve values from the future)
24
+ # return a result with no 'values' key set.
25
+ query = 'SELECT value FROM requests_per_minute WHERE time > 1437019900'
26
+ response = {'results'=>[{'series'=>[{'name'=>'requests_per_minute' ,'columns' => ['time','value']}]}]}
27
+ stub_request(:get, 'http://influxdb.test:9999/query').with(
28
+ query: { db: 'database', precision: 's', u: 'username', p: 'password', q: query }
29
+ ).to_return(body: JSON.generate(response), status: 200)
30
+ expected_result = [{'name'=>'requests_per_minute', 'tags'=>nil, 'values'=>[]}]
31
+ expect(subject.query(query)).to eq(expected_result)
32
+ end
33
+ end
34
+ end