influxdb_setup 0.3.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4471496a08494e71b0e21ef15489732db87238bf
-  data.tar.gz: 9860d614165a6b8afddf1743309d2e6cb89a9056
+  metadata.gz: af774bd0bda065750d0b284e4bfb5dbcba4e6e44
+  data.tar.gz: d28478b3420de3ed667969acd98f6644cd3714ed
 SHA512:
-  metadata.gz: 10068472722032acf450895cc5331a26362ae4180c439739f249d4a0b279b5e241a12ed18ece490c1eabd71b5fa17ea34f53610ca3c64da684c712cc9a89985c
-  data.tar.gz: a78a8c0f9c49d84737ec6ffdf2defa4f155fdefd6718aece8ac6dfadbbaf9d1d2f563009a841c43a0d6bfb459cfadf27405734368db34b630a859fff01152a03
+  metadata.gz: c287d4135601d44847afd6d98617054778600097e6ee68606a44239832516986de78bb4fd077aab66236ce112add87db143ff5e6bf47c98124b2acbb2b228a26
+  data.tar.gz: d67ffd8c67f5221ccba1633370e0ec471707de5047941fa567d1e48e63227cc92bb2250108e0cd90280ea12713c5502227aa40c9682c878775e7e6a90b126ea9

data/CHANGELOG.md

@@ -0,0 +1,14 @@
+# Change Log
+All notable changes to this project will be documented in this file.
+This project adheres to [Semantic Versioning](http://semver.org/).
+CHANGELOG inspiration from http://keepachangelog.com/.
+
+## Unreleased
+
+## [0.5.0] - March 17, 2016
+* Update influxdb_queries.yml and LoadQueries for InfluxDB 0.10
+
+## [0.4.1] - March 4, 2016
+* Skip loading continuous queries if influxdb_queries.yml file doesn't exist.
+* Use a real logger object, this will cause timestamps to be printed, and makes
+  testing easier.

data/README.md

@@ -1,19 +1,14 @@
 # InfluxdbSetup
 
-For configuring the influxdb database, shard spaces, and continuous queries.
-
-**Warning**: Only works with InfluxDB 0.8.x
-
-I wanted configuring influxdb to be as easy as database migrations with Rails.
-This gem provides rake tasks to create the database and user, setup 2 default
-shard spaces (one for a week of fine grained detail, and another for archiving
-a year of data), and for creating/removing/updating continuous queries.
+For configuring the influxdb database, and continuous queries.
 
 ## Installation
 
 Add these lines to your application's Gemfile:
 
 ```ruby
+source 'http://gems.corp.avvo.com'
+
 gem 'influxdb_setup'
 ```
 
@@ -26,10 +21,9 @@ And then execute:
 To have the task run on deploy, add `require "influxdb_setup/capistrano"` to
 your `config/deploy.rb`.
 
-This will run the setup on deploy (creating database, shard spaces, and
+This will run the setup on deploy (creating database, user, and
 continuous queries). It will also mark the deploy in the "deploys" table in
-your influxdb. See the example `db/influxdb_queries.yml` below for the archive
-queries.
+your influxdb. See the example `influxdb_queries.yml` for the archive queries.
 
 ## Usage
 
@@ -52,18 +46,29 @@ default: &default
 
 development:
   <<: *default
-  hosts: ["192.168.59.103"] # boot2docker default ip
+  hosts: ["192.168.59.103"]
   async: false
   enabled: true
   retry: false
 
+ec2:
+  <<: *default
+
 test:
   <<: *default
 
+stag:
+  <<: *default
+  hosts: ["fs2wad.prod.avvo.com"]
+  enabled: true
+
 production:
   <<: *default
-  hosts: ["localhost"]
+  hosts: ["fs2wad.prod.avvo.com"]
   enabled: true
+
+docker:
+  <<: *default
 ```
 
 To add continuous queries, just add them to the `db/influxdb_queries.yml`, for
@@ -71,10 +76,14 @@ example:
 
 ```yaml
 ---
- - select * from "response_times" into response_times.[rails_env]
- - select mean(value),count(value),percentile(value,95.0) as 95th,percentile(value,99.0) as 99th from "response_times.production" group by time(1h) into archive.response_times.1h
- - select * from "deploys" into deploys.[rails_env]
- - select * from "deploys.production" into archive.deploys
+stag_times:
+  select median(value) as median,mean(value) as mean,count(value) as count into one_hour_stag_response_times from "response_times" where rails_env='stag' group by time(1h)
+prod_times:
+  select median(value) as median,mean(value) as mean,count(value) as count into one_hour_prod_response_times from "response_times" where rails_env='production' group by time(1h)
+prod_deploys:
+  select count(commit) as deploys into prod_deploys_per_hour from "deploys" where rails_env='production' group by time(1h)
+stag_deploys:
+  select count(commit) as deploys into stag_deploys_per_hour from "deploys" where rails_env='stag' group by time(1h)
 ```
 
 Make sure your queries match what the server coerces them into (no spaces
@@ -86,10 +95,6 @@ If there's queries to update the task will not do anything.
 `rake influxdb:create_db`
 Creates the database for the service if it doesn't already exist.
 
-`rake influxdb:setup_shard_spaces`
-Creates or updates the default and archives shard spaces. If they don't exist,
-it creates them. If they do exist but they are not correct, it updates them.
-
 `rake influxdb:create_user`
 Creates the user for the service if it doesn't already exist.
 
@@ -100,16 +105,42 @@ in the `db/influxdb_queries.yml` file.
 `rake influxdb:setup`
 Runs all the above rake tasks.
 
+## Tests
+
+To run the tests, you need an influxdb host setup. If you're not running it on
+`localhost:8086`, you can specify where it's running with the `INFLUXDB_HOSTS`
+and `INFLUXDB_PORT` environmental variables.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run
 `rake spec` to run the tests. You can also run `bin/console` for an interactive
 prompt that will allow you to experiment.
 
+To install this gem onto your local machine, run `bundle exec rake install`. To
+release a new version, update the version number in `version.rb`, and then run
+`bundle exec rake release`, which will create a git tag for the version, push
+git commits and tags, and push the `.gem` file to
+[rubygems.org](https://rubygems.org).
+
+To cut a gem:
+
+1. Bump the version in `lib/influxdb_setup/version.rb`
+2. Build the gem `gem_push=no rake release`
+3. Push to geminabox `gem inabox pkg/influxdb_setup-0.1.0.gem` (or whatever
+version you just cut.)
+
 ## Contributing
 
-Bug reports and pull requests are welcome on Github at
-https://github.com/dplummer/influxdb_setup
+Bug reports and pull requests are welcome on GitLab at
+https://github.com/avvo/influxdb_setup.
+
+## Changelog
+
+v0.4.0 - Upgrade influxdb gem to handle InfluxDB v0.9.x and greater and remove shard space setup support
+v0.3.1 - automatically skip influxdb setup on a rollback
+v0.3.0 - added the ability to skip influxdb setup by setting the capistrano variable skip_influx_setup
+
 
 ## License
 

data/influxdb_setup.gemspec

@@ -7,10 +7,10 @@ Gem::Specification.new do |spec|
   spec.name          = "influxdb_setup"
   spec.version       = InfluxdbSetup::VERSION
   spec.authors       = ["Donald Plummer"]
-  spec.email         = ["donald.plummer@gmail.com"]
+  spec.email         = ["dplummer@avvo.com"]
 
   spec.summary       = %q{Rake task for setting up an influxdb database and queries}
-  spec.homepage      = "https://github.com/dplummer/influxdb_setup"
+  spec.homepage      = "https://github.com/avvo/influxdb_setup"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
@@ -18,9 +18,10 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "influxdb", "~> 0.1.9"
+  spec.add_dependency "influxdb", "~> 0.2.3"
 
   spec.add_development_dependency "bundler", "~> 1.10"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec"
+  spec.add_development_dependency "pry"
 end

data/lib/influxdb_setup/capistrano.rb

@@ -1,15 +1,23 @@
 namespace :deploy do
+  # on rollback, repo_path is not set or is not present, so we can't do
+  # the influxdb_setup task below
+  before :rollback, :skip_influxdb do
+    set :skip_influxdb_setup, true
+  end
+
   before :restart, :influxdb_setup do
     on roles(:db) do
-      within release_path do
-        revision = fetch(:current_revision) do
-          within(repo_path) do
-            capture("cd #{repo_path} && git rev-parse --short HEAD")
+      unless fetch(:skip_influxdb_setup)
+        within current_path do
+          revision = fetch(:current_revision) do
+            within(repo_path) do
+              capture("cd #{repo_path} && git rev-parse --short HEAD")
+            end
           end
-        end
 
-        with rails_env: fetch(:rails_env) do
-          execute :rake, "influxdb:setup influxdb:mark_deploy[#{revision}]"
+          with rails_env: fetch(:rails_env), newrelic_agent_enabled: false do
+            execute :rake, "influxdb:setup influxdb:mark_deploy[#{revision}]"
+          end
         end
       end
     end

data/lib/influxdb_setup/command.rb

@@ -7,7 +7,7 @@ module InfluxdbSetup
     end
 
     def log(message)
-      puts "[InfluxdbSetup] #{message}"
+      @config.logger.info "[InfluxdbSetup] #{message}"
     end
   end
 end

data/lib/influxdb_setup/commands.rb

@@ -8,7 +8,6 @@ module InfluxdbSetup
 
     {
       create_db: CreateDb,
-      setup_shard_spaces: SetupShardSpaces,
       create_user: CreateUser,
       load_queries: LoadQueries,
       mark_deploy: MarkDeploy,

data/lib/influxdb_setup/config.rb

@@ -1,5 +1,6 @@
 require 'yaml'
 require 'erb'
+require 'influxdb'
 
 module InfluxdbSetup
   class Config
@@ -11,8 +12,11 @@ module InfluxdbSetup
       @config ||= YAML.load(ERB.new(File.read("config/influxdb.yml")).result)[env]
     end
 
-    def initialize
-      @config = self.class.config
+    attr_accessor :logger
+
+    def initialize(config: self.class.config, logger: Logger.new(STDOUT))
+      @config = config
+      @logger = logger
     end
 
     def env

data/lib/influxdb_setup/create_db.rb

@@ -3,7 +3,7 @@ module InfluxdbSetup
     def call
       db = @config.db_name
       root = @config.build_client
-      databases = root.get_database_list.map { |row| row["name"] }
+      databases = root.list_databases.map { |row| row["name"] }
 
       unless databases.include?(db)
         root.create_database(db)

data/lib/influxdb_setup/create_user.rb

@@ -6,9 +6,9 @@ module InfluxdbSetup
       pass = @config.password
 
       root = @config.build_client
-      users = root.get_database_user_list(db).map { |row| row["name"] }
+      users = root.list_users.map{|user_hash| user_hash["username"]}
 
-      unless users.include?(db)
+      unless users.include?(user)
         root.create_database_user(db, user, pass)
       else
         log "Influxdb user '#{user}'@'#{db}' already exists"

data/lib/influxdb_setup/load_queries.rb

@@ -1,33 +1,52 @@
 module InfluxdbSetup
   class LoadQueries < Command
-    def call
-      db = @config.db_name
-      root = @config.build_client(db)
-      url = root.send :full_url, "/cluster/configuration"
-      configuration = root.send :get, url
+    class Query
+      attr_reader :raw_name, :query
 
-      existing_queries = configuration["ContinuousQueries"].fetch(db, {}).each_with_object({}) do |row, acc|
-        acc[row['Id']] = row['Query']
+      def initialize(raw_name, query)
+        @raw_name = raw_name
+        @query = query
       end
-      expected_queries = YAML.load_file("db/influxdb_queries.yml")
-
-      no_change = 0
-      expected_queries.each do |query|
-        unless existing_queries.values.include?(query)
-          log "Adding '#{query}'"
-          root.query query
-        else
-          no_change += 1
-        end
+
+      def name
+        "influxdb_setup_#{raw_name}"
       end
+    end
+
+    FileFormatError = Class.new(StandardError)
 
-      log "There were #{no_change} continuous queries that required no updates"
+    def call
+      queries_file = Pathname.new("db/influxdb_queries.yml")
+
+      if queries_file.exist?
+        db = @config.db_name
+        root = @config.build_client(db)
+        existing_queries = root.list_continuous_queries(db)
+        raw = YAML.load(ERB.new(queries_file.read).result) || {}
+        raise FileFormatError, "expected influxdb_queries.yml to be a hash, was a #{raw.class.name}" unless raw.is_a?(Hash)
+
+        expected_queries = raw.map do |name, query|
+          Query.new(name, query)
+        end
+
+        # Delete first in case an old name is getting overwritten
+        existing_queries.each do |query|
+          if expected_queries.none? {|expected| expected.name == query["name"]}
+            log "Removing '#{query['name']}', was: '#{query['query']}'"
+            root.delete_continuous_query(query["name"], db)
+          end
+        end
 
-      existing_queries.each do |(id, query)|
-        unless expected_queries.include?(query)
-          log "Removing '#{query}'"
-          root.query "drop continuous query #{id}"
+        expected_queries.each do |expected|
+          if existing_queries.any? {|hash| hash["name"] == expected.name}
+            log "Skipping '#{expected.raw_name}', a query by that name already exists"
+          else
+            log "Adding '#{expected.raw_name}': '#{expected.query}'"
+            root.create_continuous_query(expected.name, db, expected.query)
+          end
         end
+      else
+        log "No influxdb_queries.yml file found, skipping continuous queries setup"
       end
     end
   end

data/lib/influxdb_setup/mark_deploy.rb

@@ -4,7 +4,7 @@ module InfluxdbSetup
       db = @config.db_name
       root = @config.build_client(db)
 
-      root.write_point("deploys", {
+      root.write_point("deploys", values: {
         rails_env: config.env,
         commit: commit
       })

data/lib/influxdb_setup/version.rb

@@ -1,3 +1,3 @@
 module InfluxdbSetup
-  VERSION = "0.3.0"
+  VERSION = "1.0.0"
 end

data/lib/influxdb_setup.rb

@@ -8,7 +8,6 @@ module InfluxdbSetup
   autoload :CreateUser, "influxdb_setup/create_user"
   autoload :LoadQueries, "influxdb_setup/load_queries"
   autoload :MarkDeploy, "influxdb_setup/mark_deploy"
-  autoload :SetupShardSpaces, "influxdb_setup/setup_shard_spaces"
 
   def self.config
     InfluxdbSetup::Config.config

data/lib/tasks/influxdb_setup.rake

@@ -8,11 +8,6 @@ namespace :influxdb do
     @influxdb_setup.create_db
   end
 
-  desc "Setup the default and archives shard spaces"
-  task :setup_shard_spaces => [:config] do
-    @influxdb_setup.setup_shard_spaces
-  end
-
   desc "Creates the service's user if it doesn't exist"
   task :create_user => [:config] do
     @influxdb_setup.create_user
@@ -30,7 +25,6 @@ namespace :influxdb do
 
   desc "Run all the tasks to setup influxdb for the service"
   task :setup => [:create_db,
-                  :setup_shard_spaces,
                   :create_user,
                   :load_queries]
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: influxdb_setup
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Donald Plummer
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-10-08 00:00:00.000000000 Z
+date: 2016-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: influxdb
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.9
+        version: 0.2.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.9
+        version: 0.2.3
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -66,9 +66,23 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
-- donald.plummer@gmail.com
+- dplummer@avvo.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -76,6 +90,7 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -93,10 +108,9 @@ files:
 - lib/influxdb_setup/load_queries.rb
 - lib/influxdb_setup/mark_deploy.rb
 - lib/influxdb_setup/railtie.rb
-- lib/influxdb_setup/setup_shard_spaces.rb
 - lib/influxdb_setup/version.rb
 - lib/tasks/influxdb_setup.rake
-homepage: https://github.com/dplummer/influxdb_setup
+homepage: https://github.com/avvo/influxdb_setup
 licenses:
 - MIT
 metadata: {}
@@ -116,7 +130,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Rake task for setting up an influxdb database and queries

data/lib/influxdb_setup/setup_shard_spaces.rb

@@ -1,51 +0,0 @@
-module InfluxdbSetup
-  class SetupShardSpaces < Command
-    def call
-      db = @config.db_name
-      root = @config.build_client
-
-      expected_default_config = {
-        "name"              => "default",
-        "database"          => db,
-        "regex"             => "/.*/",
-        "retentionPolicy"   => "7d",
-        "shardDuration"     => "1h",
-        "replicationFactor" => 1,
-        "split"             => 1
-      }
-
-      expected_archive_config = {
-        "name"              => "archives",
-        "database"          => db,
-        "regex"             => "/archive.*/",
-        "retentionPolicy"   => "365d",
-        "shardDuration"     => "7d",
-        "replicationFactor" => 1,
-        "split"             => 1
-      }
-
-      actual_default_shard = root.get_shard_space(db, "default")
-      actual_archive_shard = root.get_shard_space(db, "archives")
-
-      if actual_default_shard.nil?
-        log "Creating default shard space"
-        root.create_shard_space(db, expected_default_config.except("database"))
-      elsif actual_default_shard != expected_default_config
-        log "Updating default shard space"
-        root.update_shard_space(db, "default", expected_default_config.except("database"))
-      else
-        log "Shard default up to date"
-      end
-
-      if actual_archive_shard.nil?
-        log "Creating archives shard space"
-        root.create_shard_space(db, expected_archive_config.except("database"))
-      elsif actual_archive_shard != expected_archive_config
-        log "Updating archives shard space"
-        root.update_shard_space(db, "archives", expected_archive_config.except("database"))
-      else
-        log "Shard archives up to date"
-      end
-    end
-  end
-end