influxdb_setup 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4471496a08494e71b0e21ef15489732db87238bf
+  data.tar.gz: 9860d614165a6b8afddf1743309d2e6cb89a9056
+SHA512:
+  metadata.gz: 10068472722032acf450895cc5331a26362ae4180c439739f249d4a0b279b5e241a12ed18ece490c1eabd71b5fa17ea34f53610ca3c64da684c712cc9a89985c
+  data.tar.gz: a78a8c0f9c49d84737ec6ffdf2defa4f155fdefd6718aece8ac6dfadbbaf9d1d2f563009a841c43a0d6bfb459cfadf27405734368db34b630a859fff01152a03

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.2.3
+before_install: gem install bundler -v 1.10.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in influxdb_setup.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Donald Plummer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,118 @@
+# InfluxdbSetup
+
+For configuring the influxdb database, shard spaces, and continuous queries.
+
+**Warning**: Only works with InfluxDB 0.8.x
+
+I wanted configuring influxdb to be as easy as database migrations with Rails.
+This gem provides rake tasks to create the database and user, setup 2 default
+shard spaces (one for a week of fine grained detail, and another for archiving
+a year of data), and for creating/removing/updating continuous queries.
+
+## Installation
+
+Add these lines to your application's Gemfile:
+
+```ruby
+gem 'influxdb_setup'
+```
+
+And then execute:
+
+    $ bundle
+
+## Capistrano Integration
+
+To have the task run on deploy, add `require "influxdb_setup/capistrano"` to
+your `config/deploy.rb`.
+
+This will run the setup on deploy (creating database, shard spaces, and
+continuous queries). It will also mark the deploy in the "deploys" table in
+your influxdb. See the example `db/influxdb_queries.yml` below for the archive
+queries.
+
+## Usage
+
+This library expects your influxdb config to be located in the
+`config/influxdb.yml` file. For example (change *myapp* to your application
+name):
+
+```yaml
+default: &default
+  hosts: <%= ENV.fetch('INFLUXDB_HOSTS', '').split(',') %>
+  port: <%= ENV.fetch('INFLUXDB_PORT', '8086') %>
+  db_name: <%= ENV.fetch('INFLUXDB_DB_NAME', 'myapp') %>
+  username: <%= ENV.fetch('INFLUXDB_USER', 'myapp') %>
+  password: <%= ENV.fetch('INFLUXDB_PASS', 'myapp') %>
+  async: <%= ENV.fetch('INFLUXDB_ASYNC', 'true') == "true" %>       # default true
+  retry: <%= ENV.fetch('INFLUXDB_RETRY', 'true') == "true" %>       # default true
+  use_ssl: <%= ENV.fetch('INFLUXDB_USE_SSL', 'false') == "true" %>  # default false
+  enabled: <%= ENV.fetch('INFLUXDB_ENABLED', 'false') == "true" %>  # default false
+
+
+development:
+  <<: *default
+  hosts: ["192.168.59.103"] # boot2docker default ip
+  async: false
+  enabled: true
+  retry: false
+
+test:
+  <<: *default
+
+production:
+  <<: *default
+  hosts: ["localhost"]
+  enabled: true
+```
+
+To add continuous queries, just add them to the `db/influxdb_queries.yml`, for
+example:
+
+```yaml
+---
+ - select * from "response_times" into response_times.[rails_env]
+ - select mean(value),count(value),percentile(value,95.0) as 95th,percentile(value,99.0) as 99th from "response_times.production" group by time(1h) into archive.response_times.1h
+ - select * from "deploys" into deploys.[rails_env]
+ - select * from "deploys.production" into archive.deploys
+```
+
+Make sure your queries match what the server coerces them into (no spaces
+after commas) by running the `rake influxdb:load_queries` task multiple times.
+If there's queries to update the task will not do anything.
+
+## Rake tasks
+
+`rake influxdb:create_db`
+Creates the database for the service if it doesn't already exist.
+
+`rake influxdb:setup_shard_spaces`
+Creates or updates the default and archives shard spaces. If they don't exist,
+it creates them. If they do exist but they are not correct, it updates them.
+
+`rake influxdb:create_user`
+Creates the user for the service if it doesn't already exist.
+
+`rake influxdb:load_queries`
+Creates any continuous queries that are missing. Removes queries that are not
+in the `db/influxdb_queries.yml` file.
+
+`rake influxdb:setup`
+Runs all the above rake tasks.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run
+`rake spec` to run the tests. You can also run `bin/console` for an interactive
+prompt that will allow you to experiment.
+
+## Contributing
+
+Bug reports and pull requests are welcome on Github at
+https://github.com/dplummer/influxdb_setup
+
+## License
+
+The gem is available as open source under the terms of the
+[MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "influxdb_setup"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/influxdb_setup.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'influxdb_setup/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "influxdb_setup"
+  spec.version       = InfluxdbSetup::VERSION
+  spec.authors       = ["Donald Plummer"]
+  spec.email         = ["donald.plummer@gmail.com"]
+
+  spec.summary       = %q{Rake task for setting up an influxdb database and queries}
+  spec.homepage      = "https://github.com/dplummer/influxdb_setup"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "influxdb", "~> 0.1.9"
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+end

data/lib/influxdb_setup/capistrano.rb

@@ -0,0 +1,17 @@
+namespace :deploy do
+  before :restart, :influxdb_setup do
+    on roles(:db) do
+      within release_path do
+        revision = fetch(:current_revision) do
+          within(repo_path) do
+            capture("cd #{repo_path} && git rev-parse --short HEAD")
+          end
+        end
+
+        with rails_env: fetch(:rails_env) do
+          execute :rake, "influxdb:setup influxdb:mark_deploy[#{revision}]"
+        end
+      end
+    end
+  end
+end

data/lib/influxdb_setup/command.rb

@@ -0,0 +1,13 @@
+module InfluxdbSetup
+  class Command
+    attr_reader :config
+
+    def initialize(config)
+      @config = config
+    end
+
+    def log(message)
+      puts "[InfluxdbSetup] #{message}"
+    end
+  end
+end

data/lib/influxdb_setup/commands.rb

@@ -0,0 +1,25 @@
+module InfluxdbSetup
+  class Commands
+    attr_reader :config
+
+    def initialize
+      @config = Config.new
+    end
+
+    {
+      create_db: CreateDb,
+      setup_shard_spaces: SetupShardSpaces,
+      create_user: CreateUser,
+      load_queries: LoadQueries,
+      mark_deploy: MarkDeploy,
+    }.each do |cmd, klass|
+      define_method cmd do |*args|
+        begin
+          klass.new(config).call(*args) if config.enabled?
+        rescue InfluxDB::ConnectionError => e
+          puts "[InfluxdbSetup##{cmd}] Skipping... #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/influxdb_setup/config.rb

@@ -0,0 +1,51 @@
+require 'yaml'
+require 'erb'
+
+module InfluxdbSetup
+  class Config
+    class << self
+      attr_writer :config
+    end
+
+    def self.config
+      @config ||= YAML.load(ERB.new(File.read("config/influxdb.yml")).result)[env]
+    end
+
+    def initialize
+      @config = self.class.config
+    end
+
+    def env
+      defined?(Rails) ? Rails.env : ENV.fetch('RAILS_ENV', 'development')
+    end
+
+    def db_name
+      @config['db_name']
+    end
+
+    def enabled?
+      @config['enabled']
+    end
+
+    def username
+      @config['username']
+    end
+
+    def password
+      @config['password']
+    end
+
+    def build_client(database = "", options = {})
+      InfluxDB::Client.new(database,
+                          {
+                            username: "root",
+                            password: "root",
+                            hosts:    @config["hosts"],
+                            port:     @config.fetch("port", 8086),
+                            async:    false,
+                            use_ssl:  @config.fetch("use_ssl", false),
+                            retry:    false,
+                          }.merge(options))
+    end
+  end
+end

data/lib/influxdb_setup/create_db.rb

@@ -0,0 +1,15 @@
+module InfluxdbSetup
+  class CreateDb < Command
+    def call
+      db = @config.db_name
+      root = @config.build_client
+      databases = root.get_database_list.map { |row| row["name"] }
+
+      unless databases.include?(db)
+        root.create_database(db)
+      else
+        log "Influxdb database '#{db}' already exists"
+      end
+    end
+  end
+end

data/lib/influxdb_setup/create_user.rb

@@ -0,0 +1,18 @@
+module InfluxdbSetup
+  class CreateUser < Command
+    def call
+      db = @config.db_name
+      user = @config.username
+      pass = @config.password
+
+      root = @config.build_client
+      users = root.get_database_user_list(db).map { |row| row["name"] }
+
+      unless users.include?(db)
+        root.create_database_user(db, user, pass)
+      else
+        log "Influxdb user '#{user}'@'#{db}' already exists"
+      end
+    end
+  end
+end

data/lib/influxdb_setup/load_queries.rb

@@ -0,0 +1,34 @@
+module InfluxdbSetup
+  class LoadQueries < Command
+    def call
+      db = @config.db_name
+      root = @config.build_client(db)
+      url = root.send :full_url, "/cluster/configuration"
+      configuration = root.send :get, url
+
+      existing_queries = configuration["ContinuousQueries"].fetch(db, {}).each_with_object({}) do |row, acc|
+        acc[row['Id']] = row['Query']
+      end
+      expected_queries = YAML.load_file("db/influxdb_queries.yml")
+
+      no_change = 0
+      expected_queries.each do |query|
+        unless existing_queries.values.include?(query)
+          log "Adding '#{query}'"
+          root.query query
+        else
+          no_change += 1
+        end
+      end
+
+      log "There were #{no_change} continuous queries that required no updates"
+
+      existing_queries.each do |(id, query)|
+        unless expected_queries.include?(query)
+          log "Removing '#{query}'"
+          root.query "drop continuous query #{id}"
+        end
+      end
+    end
+  end
+end

data/lib/influxdb_setup/mark_deploy.rb

@@ -0,0 +1,15 @@
+module InfluxdbSetup
+  class MarkDeploy < Command
+    def call(commit)
+      db = @config.db_name
+      root = @config.build_client(db)
+
+      root.write_point("deploys", {
+        rails_env: config.env,
+        commit: commit
+      })
+
+      log("Marked deploy: #{config.env} at sha #{commit}")
+    end
+  end
+end

data/lib/influxdb_setup/railtie.rb

@@ -0,0 +1,7 @@
+module InfluxdbSetup
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load "tasks/influxdb_setup.rake"
+    end
+  end
+end

data/lib/influxdb_setup/setup_shard_spaces.rb

@@ -0,0 +1,51 @@
+module InfluxdbSetup
+  class SetupShardSpaces < Command
+    def call
+      db = @config.db_name
+      root = @config.build_client
+
+      expected_default_config = {
+        "name"              => "default",
+        "database"          => db,
+        "regex"             => "/.*/",
+        "retentionPolicy"   => "7d",
+        "shardDuration"     => "1h",
+        "replicationFactor" => 1,
+        "split"             => 1
+      }
+
+      expected_archive_config = {
+        "name"              => "archives",
+        "database"          => db,
+        "regex"             => "/archive.*/",
+        "retentionPolicy"   => "365d",
+        "shardDuration"     => "7d",
+        "replicationFactor" => 1,
+        "split"             => 1
+      }
+
+      actual_default_shard = root.get_shard_space(db, "default")
+      actual_archive_shard = root.get_shard_space(db, "archives")
+
+      if actual_default_shard.nil?
+        log "Creating default shard space"
+        root.create_shard_space(db, expected_default_config.except("database"))
+      elsif actual_default_shard != expected_default_config
+        log "Updating default shard space"
+        root.update_shard_space(db, "default", expected_default_config.except("database"))
+      else
+        log "Shard default up to date"
+      end
+
+      if actual_archive_shard.nil?
+        log "Creating archives shard space"
+        root.create_shard_space(db, expected_archive_config.except("database"))
+      elsif actual_archive_shard != expected_archive_config
+        log "Updating archives shard space"
+        root.update_shard_space(db, "archives", expected_archive_config.except("database"))
+      else
+        log "Shard archives up to date"
+      end
+    end
+  end
+end

data/lib/influxdb_setup/version.rb

@@ -0,0 +1,3 @@
+module InfluxdbSetup
+  VERSION = "0.3.0"
+end

data/lib/influxdb_setup.rb

@@ -0,0 +1,22 @@
+require "influxdb_setup/version"
+
+module InfluxdbSetup
+  autoload :Command, "influxdb_setup/command"
+  autoload :Commands, "influxdb_setup/commands"
+  autoload :Config, "influxdb_setup/config"
+  autoload :CreateDb, "influxdb_setup/create_db"
+  autoload :CreateUser, "influxdb_setup/create_user"
+  autoload :LoadQueries, "influxdb_setup/load_queries"
+  autoload :MarkDeploy, "influxdb_setup/mark_deploy"
+  autoload :SetupShardSpaces, "influxdb_setup/setup_shard_spaces"
+
+  def self.config
+    InfluxdbSetup::Config.config
+  end
+
+  def self.config=(new_config)
+    InfluxdbSetup::Config.config = new_config
+  end
+end
+
+require "influxdb_setup/railtie" if defined?(Rails)

data/lib/tasks/influxdb_setup.rake

@@ -0,0 +1,36 @@
+namespace :influxdb do
+  task :config => [:environment] do
+    @influxdb_setup = InfluxdbSetup::Commands.new
+  end
+
+  desc "Creates the influxdb database unless it already exists"
+  task :create_db => [:config] do
+    @influxdb_setup.create_db
+  end
+
+  desc "Setup the default and archives shard spaces"
+  task :setup_shard_spaces => [:config] do
+    @influxdb_setup.setup_shard_spaces
+  end
+
+  desc "Creates the service's user if it doesn't exist"
+  task :create_user => [:config] do
+    @influxdb_setup.create_user
+  end
+
+  desc "Loads the continuous queries from db/influxdb_queries.yml"
+  task :load_queries => [:config] do
+    @influxdb_setup.load_queries
+  end
+
+  desc "Log deploy in influxdb"
+  task :mark_deploy, [:commit] => [:config] do |t, args|
+    @influxdb_setup.mark_deploy(args[:commit])
+  end
+
+  desc "Run all the tasks to setup influxdb for the service"
+  task :setup => [:create_db,
+                  :setup_shard_spaces,
+                  :create_user,
+                  :load_queries]
+end

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: influxdb_setup
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+platform: ruby
+authors:
+- Donald Plummer
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-10-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: influxdb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.9
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.9
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- donald.plummer@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- influxdb_setup.gemspec
+- lib/influxdb_setup.rb
+- lib/influxdb_setup/capistrano.rb
+- lib/influxdb_setup/command.rb
+- lib/influxdb_setup/commands.rb
+- lib/influxdb_setup/config.rb
+- lib/influxdb_setup/create_db.rb
+- lib/influxdb_setup/create_user.rb
+- lib/influxdb_setup/load_queries.rb
+- lib/influxdb_setup/mark_deploy.rb
+- lib/influxdb_setup/railtie.rb
+- lib/influxdb_setup/setup_shard_spaces.rb
+- lib/influxdb_setup/version.rb
+- lib/tasks/influxdb_setup.rake
+homepage: https://github.com/dplummer/influxdb_setup
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: Rake task for setting up an influxdb database and queries
+test_files: []