influxdb 0.1.9 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.rubocop.yml +41 -0
- data/.travis.yml +3 -2
- data/Gemfile +7 -1
- data/README.md +218 -102
- data/Rakefile +2 -6
- data/lib/influxdb.rb +15 -5
- data/lib/influxdb/client.rb +38 -433
- data/lib/influxdb/client/http.rb +123 -0
- data/lib/influxdb/config.rb +66 -0
- data/lib/influxdb/errors.rb +8 -2
- data/lib/influxdb/{logger.rb → logging.rb} +6 -5
- data/lib/influxdb/max_queue.rb +2 -1
- data/lib/influxdb/point_value.rb +27 -25
- data/lib/influxdb/query/cluster.rb +17 -0
- data/lib/influxdb/query/continuous_query.rb +22 -0
- data/lib/influxdb/query/core.rb +110 -0
- data/lib/influxdb/query/database.rb +21 -0
- data/lib/influxdb/query/retention_policy.rb +26 -0
- data/lib/influxdb/query/user.rb +41 -0
- data/lib/influxdb/version.rb +2 -2
- data/lib/influxdb/writer/async.rb +115 -0
- data/lib/influxdb/writer/udp.rb +21 -0
- data/spec/influxdb/cases/async_client_spec.rb +33 -0
- data/spec/influxdb/cases/query_cluster_spec.rb +65 -0
- data/spec/influxdb/cases/query_continuous_query_spec.rb +82 -0
- data/spec/influxdb/cases/query_core.rb +34 -0
- data/spec/influxdb/cases/query_database_spec.rb +58 -0
- data/spec/influxdb/cases/query_retention_policy_spec.rb +84 -0
- data/spec/influxdb/cases/query_series_spec.rb +50 -0
- data/spec/influxdb/cases/query_shard_space_spec.rb +105 -0
- data/spec/influxdb/cases/query_shard_spec.rb +43 -0
- data/spec/influxdb/cases/query_user_spec.rb +127 -0
- data/spec/influxdb/cases/querying_spec.rb +149 -0
- data/spec/influxdb/cases/retry_requests_spec.rb +102 -0
- data/spec/influxdb/cases/udp_client_spec.rb +21 -0
- data/spec/influxdb/cases/write_points_spec.rb +140 -0
- data/spec/influxdb/client_spec.rb +37 -810
- data/spec/influxdb/config_spec.rb +118 -0
- data/spec/influxdb/{logger_spec.rb → logging_spec.rb} +4 -8
- data/spec/influxdb/max_queue_spec.rb +29 -0
- data/spec/influxdb/point_value_spec.rb +81 -14
- data/spec/influxdb/worker_spec.rb +8 -11
- data/spec/spec_helper.rb +7 -10
- metadata +65 -30
- data/lib/influxdb/udp_client.rb +0 -16
- data/lib/influxdb/worker.rb +0 -80
- data/spec/influxdb/udp_client_spec.rb +0 -33
- data/spec/influxdb_spec.rb +0 -4
- data/spec/max_queue_spec.rb +0 -32
@@ -0,0 +1,21 @@
|
|
1
|
+
module InfluxDB
|
2
|
+
module Query
|
3
|
+
module Database # :nodoc:
|
4
|
+
def create_database(name)
|
5
|
+
execute("CREATE DATABASE #{name}")
|
6
|
+
end
|
7
|
+
|
8
|
+
def delete_database(name)
|
9
|
+
execute("DROP DATABASE #{name}")
|
10
|
+
end
|
11
|
+
|
12
|
+
def list_databases
|
13
|
+
resp = execute("SHOW DATABASES", parse: true)
|
14
|
+
fetch_series(resp).fetch(0, {})
|
15
|
+
.fetch('values', [])
|
16
|
+
.flatten
|
17
|
+
.map { |v| { 'name' => v } }
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module InfluxDB
|
2
|
+
module Query
|
3
|
+
module RetentionPolicy # :nodoc:
|
4
|
+
def create_retention_policy(name, database, duration, replication, default = false)
|
5
|
+
execute(
|
6
|
+
"CREATE RETENTION POLICY \"#{name}\" ON #{database} "\
|
7
|
+
"DURATION #{duration} REPLICATION #{replication}#{default ? ' DEFAULT' : ''}")
|
8
|
+
end
|
9
|
+
|
10
|
+
def list_retention_policies(database)
|
11
|
+
resp = execute("SHOW RETENTION POLICIES \"#{database}\"", parse: true)
|
12
|
+
data = fetch_series(resp).fetch(0)
|
13
|
+
|
14
|
+
data['values'].map do |policy|
|
15
|
+
policy.each.with_index.inject({}) do |hash, (value, index)|
|
16
|
+
hash.tap { |h| h[data['columns'][index]] = value }
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def delete_retention_policy(name, database)
|
22
|
+
execute("DROP RETENTION POLICY \"#{name}\" ON #{database}")
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module InfluxDB
|
2
|
+
module Query
|
3
|
+
module User # :nodoc:
|
4
|
+
# create_database_user('testdb', 'user', 'pass') - grants all privileges by default
|
5
|
+
# create_database_user('testdb', 'user', 'pass', permissions: :read) - use [:read|:write|:all]
|
6
|
+
def create_database_user(database, username, password, options = {})
|
7
|
+
permissions = options.fetch(:permissions, :all)
|
8
|
+
execute(
|
9
|
+
"CREATE user #{username} WITH PASSWORD '#{password}'; "\
|
10
|
+
"GRANT #{permissions.to_s.upcase} ON #{database} TO #{username}"
|
11
|
+
)
|
12
|
+
end
|
13
|
+
|
14
|
+
def update_user_password(username, password)
|
15
|
+
execute("SET PASSWORD FOR #{username} = '#{password}'")
|
16
|
+
end
|
17
|
+
|
18
|
+
# permission => [:read|:write|:all]
|
19
|
+
def grant_user_privileges(username, database, permission)
|
20
|
+
execute("GRANT #{permission.to_s.upcase} ON #{database} TO #{username}")
|
21
|
+
end
|
22
|
+
|
23
|
+
# permission => [:read|:write|:all]
|
24
|
+
def revoke_user_privileges(username, database, permission)
|
25
|
+
execute("REVOKE #{permission.to_s.upcase} ON #{database} FROM #{username}")
|
26
|
+
end
|
27
|
+
|
28
|
+
def delete_user(username)
|
29
|
+
execute("DROP USER #{username}")
|
30
|
+
end
|
31
|
+
|
32
|
+
# => [{"username"=>"usr", "admin"=>true}, {"username"=>"justauser", "admin"=>false}]
|
33
|
+
def list_users
|
34
|
+
resp = execute("SHOW USERS", parse: true)
|
35
|
+
fetch_series(resp).fetch(0, {})
|
36
|
+
.fetch('values', [])
|
37
|
+
.map { |v| { 'username' => v.first, 'admin' => v.last } }
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
data/lib/influxdb/version.rb
CHANGED
@@ -1,3 +1,3 @@
|
|
1
|
-
module InfluxDB
|
2
|
-
VERSION = "0.
|
1
|
+
module InfluxDB # :nodoc:
|
2
|
+
VERSION = "0.2.0"
|
3
3
|
end
|
@@ -0,0 +1,115 @@
|
|
1
|
+
require 'thread'
|
2
|
+
require "net/http"
|
3
|
+
require "uri"
|
4
|
+
|
5
|
+
module InfluxDB
|
6
|
+
module Writer # :nodoc: all
|
7
|
+
class Async
|
8
|
+
attr_reader :config, :client
|
9
|
+
|
10
|
+
def initialize(client, config)
|
11
|
+
@client = client
|
12
|
+
@config = config
|
13
|
+
end
|
14
|
+
|
15
|
+
def write(data, _precision = nil)
|
16
|
+
data = data.is_a?(Array) ? data : [data]
|
17
|
+
data.map { |p| worker.push(p) }
|
18
|
+
end
|
19
|
+
|
20
|
+
WORKER_MUTEX = Mutex.new
|
21
|
+
def worker
|
22
|
+
return @worker if @worker
|
23
|
+
WORKER_MUTEX.synchronize do
|
24
|
+
# this return is necessary because the previous mutex holder
|
25
|
+
# might have already assigned the @worker
|
26
|
+
return @worker if @worker
|
27
|
+
@worker = Worker.new(client, config)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
class Worker
|
32
|
+
attr_reader :client, :queue, :threads
|
33
|
+
|
34
|
+
include InfluxDB::Logging
|
35
|
+
|
36
|
+
MAX_POST_POINTS = 1000
|
37
|
+
MAX_QUEUE_SIZE = 10_000
|
38
|
+
NUM_WORKER_THREADS = 3
|
39
|
+
SLEEP_INTERVAL = 5
|
40
|
+
|
41
|
+
def initialize(client, config)
|
42
|
+
@client = client
|
43
|
+
config = config.is_a?(Hash) ? config : {}
|
44
|
+
@queue = InfluxDB::MaxQueue.new config.fetch(:max_queue, MAX_QUEUE_SIZE)
|
45
|
+
|
46
|
+
spawn_threads!
|
47
|
+
|
48
|
+
at_exit do
|
49
|
+
log :debug, "Thread exiting, flushing queue."
|
50
|
+
check_background_queue until queue.empty?
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def push(payload)
|
55
|
+
queue.push(payload)
|
56
|
+
end
|
57
|
+
|
58
|
+
def current_threads
|
59
|
+
Thread.list.select { |t| t[:influxdb] == object_id }
|
60
|
+
end
|
61
|
+
|
62
|
+
def current_thread_count
|
63
|
+
Thread.list.count { |t| t[:influxdb] == object_id }
|
64
|
+
end
|
65
|
+
|
66
|
+
# rubocop:disable Metrics/CyclomaticComplexity
|
67
|
+
# rubocop:disable Metrics/MethodLength
|
68
|
+
# rubocop:disable Metrics/AbcSize
|
69
|
+
|
70
|
+
def spawn_threads!
|
71
|
+
@threads = []
|
72
|
+
NUM_WORKER_THREADS.times do |thread_num|
|
73
|
+
log :debug, "Spawning background worker thread #{thread_num}."
|
74
|
+
|
75
|
+
@threads << Thread.new do
|
76
|
+
Thread.current[:influxdb] = object_id
|
77
|
+
|
78
|
+
until client.stopped?
|
79
|
+
check_background_queue(thread_num)
|
80
|
+
sleep rand(SLEEP_INTERVAL)
|
81
|
+
end
|
82
|
+
|
83
|
+
log :debug, "Exit background worker thread #{thread_num}."
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
def check_background_queue(thread_num = 0)
|
89
|
+
log :debug,
|
90
|
+
"Checking background queue on thread #{thread_num} (#{current_thread_count} active)"
|
91
|
+
|
92
|
+
loop do
|
93
|
+
data = []
|
94
|
+
|
95
|
+
while data.size < MAX_POST_POINTS && !queue.empty?
|
96
|
+
p = queue.pop(true) rescue next
|
97
|
+
data.push p
|
98
|
+
end
|
99
|
+
|
100
|
+
return if data.empty?
|
101
|
+
|
102
|
+
begin
|
103
|
+
log :debug, "Found data in the queue! (#{data.length} points)"
|
104
|
+
client.write(data)
|
105
|
+
rescue => e
|
106
|
+
puts "Cannot write data: #{e.inspect}"
|
107
|
+
end
|
108
|
+
|
109
|
+
break if queue.length > MAX_POST_POINTS
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module InfluxDB
|
2
|
+
module Writer
|
3
|
+
# Writes data to InfluxDB through UDP
|
4
|
+
class UDP
|
5
|
+
attr_accessor :socket
|
6
|
+
attr_reader :host, :port
|
7
|
+
def initialize(client, config)
|
8
|
+
@client = client
|
9
|
+
config = config.is_a?(Hash) ? config : {}
|
10
|
+
@host = config.fetch(:host, 'localhost')
|
11
|
+
@port = config.fetch(:port, 4444)
|
12
|
+
self.socket = UDPSocket.new
|
13
|
+
socket.connect(host, port)
|
14
|
+
end
|
15
|
+
|
16
|
+
def write(payload, _precision = nil)
|
17
|
+
socket.send(payload, 0)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "timeout"
|
3
|
+
|
4
|
+
describe InfluxDB::Client do
|
5
|
+
let(:subject) { described_class.new(async: true) }
|
6
|
+
|
7
|
+
let(:worker_klass) { InfluxDB::Writer::Async::Worker }
|
8
|
+
|
9
|
+
specify { expect(subject.writer).to be_a(InfluxDB::Writer::Async) }
|
10
|
+
|
11
|
+
describe "#write_point" do
|
12
|
+
let(:payload) { "responses,region=eu value=5" }
|
13
|
+
|
14
|
+
it "sends writes to client" do
|
15
|
+
# exact times can be 2 or 3 (because we have 3 worker threads),
|
16
|
+
# but cannot be less than 2 due to MAX_POST_POINTS limit
|
17
|
+
expect(subject).to(receive(:write)).at_least(2).times
|
18
|
+
|
19
|
+
(worker_klass::MAX_POST_POINTS + 100).times do
|
20
|
+
subject.write_point('a', {})
|
21
|
+
end
|
22
|
+
|
23
|
+
Timeout.timeout(2 * worker_klass::SLEEP_INTERVAL) do
|
24
|
+
subject.stop!
|
25
|
+
# ensure threads exit
|
26
|
+
subject.writer.worker.threads.each(&:join)
|
27
|
+
|
28
|
+
# flush queue (we cannot test `at_exit`)
|
29
|
+
subject.writer.worker.check_background_queue
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "json"
|
3
|
+
|
4
|
+
describe InfluxDB::Client do
|
5
|
+
let(:subject) do
|
6
|
+
described_class.new(
|
7
|
+
"database",
|
8
|
+
{
|
9
|
+
host: "influxdb.test",
|
10
|
+
port: 9999,
|
11
|
+
username: "username",
|
12
|
+
password: "password",
|
13
|
+
time_precision: "s"
|
14
|
+
}.merge(args)
|
15
|
+
)
|
16
|
+
end
|
17
|
+
|
18
|
+
let(:args) { {} }
|
19
|
+
|
20
|
+
describe "#create_cluster_admin" do
|
21
|
+
let(:user) { 'adminadmin' }
|
22
|
+
let(:pass) { 'passpass' }
|
23
|
+
let(:query) { "CREATE USER #{user} WITH PASSWORD '#{pass}' WITH ALL PRIVILEGES" }
|
24
|
+
|
25
|
+
before do
|
26
|
+
stub_request(:get, "http://influxdb.test:9999/query").with(
|
27
|
+
query: { u: "username", p: "password", q: query }
|
28
|
+
)
|
29
|
+
end
|
30
|
+
|
31
|
+
it "should GET to create a new cluster admin" do
|
32
|
+
expect(subject.create_cluster_admin(user, pass)).to be_a(Net::HTTPOK)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
describe "#list_cluster_admins" do
|
37
|
+
let(:response) { { "results" => [{ "series" => [{ "columns" => %w(user admin), "values" => [["dbadmin", true], ["foobar", false]] }] }] } }
|
38
|
+
let(:expected_result) { ["dbadmin"] }
|
39
|
+
|
40
|
+
before do
|
41
|
+
stub_request(:get, "http://influxdb.test:9999/query").with(
|
42
|
+
query: { u: "username", p: "password", q: "SHOW USERS" }
|
43
|
+
).to_return(body: JSON.generate(response, status: 200))
|
44
|
+
end
|
45
|
+
|
46
|
+
it "should GET a list of cluster admins" do
|
47
|
+
expect(subject.list_cluster_admins).to eq(expected_result)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
describe "#revoke_cluster_admin_privileges" do
|
52
|
+
let(:user) { 'useruser' }
|
53
|
+
let(:query) { "REVOKE ALL PRIVILEGES FROM #{user}" }
|
54
|
+
|
55
|
+
before do
|
56
|
+
stub_request(:get, "http://influxdb.test:9999/query").with(
|
57
|
+
query: { u: "username", p: "password", q: query }
|
58
|
+
)
|
59
|
+
end
|
60
|
+
|
61
|
+
it "should GET to revoke cluster admin privileges from a user" do
|
62
|
+
expect(subject.revoke_cluster_admin_privileges(user)).to be_a(Net::HTTPOK)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "json"
|
3
|
+
|
4
|
+
describe InfluxDB::Client do
|
5
|
+
let(:subject) do
|
6
|
+
described_class.new(
|
7
|
+
"database",
|
8
|
+
{
|
9
|
+
host: "influxdb.test",
|
10
|
+
port: 9999,
|
11
|
+
username: "username",
|
12
|
+
password: "password",
|
13
|
+
time_precision: "s"
|
14
|
+
}.merge(args)
|
15
|
+
)
|
16
|
+
end
|
17
|
+
|
18
|
+
let(:args) { {} }
|
19
|
+
|
20
|
+
describe "#list_continuous_queries" do
|
21
|
+
let(:query) { "SHOW CONTINUOUS QUERIES" }
|
22
|
+
let(:database) { "testdb" }
|
23
|
+
let(:response) do
|
24
|
+
{ "results" => [{ "series" => [{ "name" => "otherdb", "columns" => %w(name query),
|
25
|
+
"values" =>
|
26
|
+
[["clicks_per_hour", "CREATE CONTINUOUS QUERY clicks_per_hour ON otherdb BEGIN SELECT count(name) INTO \"otherdb\".\"default\".clicksCount_1h FROM \"otherdb\".\"default\".clicks GROUP BY time(1h) END"]] },
|
27
|
+
{ "name" => "testdb", "columns" => %w(name query),
|
28
|
+
"values" =>
|
29
|
+
[["event_counts", "CREATE CONTINUOUS QUERY event_counts ON testdb BEGIN SELECT count(type) INTO \"testdb\".\"default\".typeCount_10m_byType FROM \"testdb\".\"default\".events GROUP BY time(10m), type END"]] }] }] }
|
30
|
+
end
|
31
|
+
|
32
|
+
let(:expected_result) do
|
33
|
+
[{ "name" => "event_counts", "query" => "CREATE CONTINUOUS QUERY event_counts ON testdb BEGIN SELECT count(type) INTO \"testdb\".\"default\".typeCount_10m_byType FROM \"testdb\".\"default\".events GROUP BY time(10m), type END" }]
|
34
|
+
end
|
35
|
+
|
36
|
+
before do
|
37
|
+
stub_request(:get, "http://influxdb.test:9999/query").with(
|
38
|
+
query: { u: "username", p: "password", q: query }
|
39
|
+
).to_return(body: JSON.generate(response), status: 200)
|
40
|
+
end
|
41
|
+
|
42
|
+
it "should GET a list of continuous queries for specified db only" do
|
43
|
+
expect(subject.list_continuous_queries(database)).to eq(expected_result)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
describe "#create_continuous_query" do
|
48
|
+
let(:name) { "event_counts_per_10m_by_type" }
|
49
|
+
let(:database) { "testdb" }
|
50
|
+
let(:query) do
|
51
|
+
"SELECT COUNT(type) INTO typeCount_10m_byType FROM events GROUP BY time(10m), type"
|
52
|
+
end
|
53
|
+
let(:clause) do
|
54
|
+
["CREATE CONTINUOUS QUERY #{name} ON #{database} BEGIN", query, "END"].join("\n")
|
55
|
+
end
|
56
|
+
|
57
|
+
before do
|
58
|
+
stub_request(:get, "http://influxdb.test:9999/query").with(
|
59
|
+
query: { u: "username", p: "password", q: clause }
|
60
|
+
)
|
61
|
+
end
|
62
|
+
|
63
|
+
it "should GET to create a new continuous query" do
|
64
|
+
expect(subject.create_continuous_query(name, database, query)).to be_a(Net::HTTPOK)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
describe "#delete_continuous_query" do
|
69
|
+
let(:name) { "event_counts_per_10m_by_type" }
|
70
|
+
let(:database) { "testdb" }
|
71
|
+
let(:query) { "DROP CONTINUOUS QUERY #{name} ON #{database}" }
|
72
|
+
before do
|
73
|
+
stub_request(:get, "http://influxdb.test:9999/query").with(
|
74
|
+
query: { u: "username", p: "password", q: query }
|
75
|
+
)
|
76
|
+
end
|
77
|
+
|
78
|
+
it "should GET to remove continuous query" do
|
79
|
+
expect(subject.delete_continuous_query(name, database)).to be_a(Net::HTTPOK)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'json'
|
3
|
+
|
4
|
+
describe InfluxDB::Client do
|
5
|
+
let(:subject) do
|
6
|
+
described_class.new(
|
7
|
+
'database',
|
8
|
+
{
|
9
|
+
host: 'influxdb.test',
|
10
|
+
port: 9999,
|
11
|
+
username: 'username',
|
12
|
+
password: 'password',
|
13
|
+
time_precision: 's'
|
14
|
+
}.merge(args)
|
15
|
+
)
|
16
|
+
end
|
17
|
+
|
18
|
+
let(:args) { {} }
|
19
|
+
|
20
|
+
describe '#query' do
|
21
|
+
|
22
|
+
it 'should handle responses with no values' do
|
23
|
+
# Some requests (such as trying to retrieve values from the future)
|
24
|
+
# return a result with no 'values' key set.
|
25
|
+
query = 'SELECT value FROM requests_per_minute WHERE time > 1437019900'
|
26
|
+
response = {'results'=>[{'series'=>[{'name'=>'requests_per_minute' ,'columns' => ['time','value']}]}]}
|
27
|
+
stub_request(:get, 'http://influxdb.test:9999/query').with(
|
28
|
+
query: { db: 'database', precision: 's', u: 'username', p: 'password', q: query }
|
29
|
+
).to_return(body: JSON.generate(response), status: 200)
|
30
|
+
expected_result = [{'name'=>'requests_per_minute', 'tags'=>nil, 'values'=>[]}]
|
31
|
+
expect(subject.query(query)).to eq(expected_result)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|