db-rotator 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +36 -20
- data/Rakefile +2 -0
- data/bin/db-rotator +0 -1
- data/db-rotator.gemspec +2 -2
- data/lib/db_rotator.rb +5 -1
- data/lib/db_rotator_config.rb +35 -8
- data/test/fixtures/basic_dump.sql +54 -0
- data/test/fixtures/basic_dump.sql.bz2 +0 -0
- data/test/fixtures/basic_dump.sql.gz +0 -0
- data/test/fixtures/basic_dump.sql.zip +0 -0
- data/test/helper.rb +19 -0
- data/test/test_db_rotator.rb +41 -2
- metadata +15 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 07ce9c6a6b6f1da73a5e9e232173748c81f84673
|
4
|
+
data.tar.gz: cc3c9b96fe45aa4f715f92c3f4e22cd928e8d695
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1d5a98d23de5eb850f8232e38beb2585f5eb2ade0eb91bb74a4b7562b53b51145451c6f7800fdeeff213f3f4c6e06cec5753b17a4efad43c745cbc6a9264d403
|
7
|
+
data.tar.gz: 883f046aec07acdddbee6cc09f3c9646674c380e910a0f655c5d028f6630f6533b9e6f716fd9b03c5725ec53bb08af50930c2f804e2d95652fd078e998cf6b84
|
data/README.md
CHANGED
@@ -1,42 +1,58 @@
|
|
1
1
|
# DBRotator
|
2
|
-
Easy MySQL database rotation and pruning.
|
2
|
+
Easy MySQL database rotation and pruning -- downloads and imports a mysql dump, and then deletes all but N databases, keeping only the newest. This tool is geared toward rotating out achived MySQL backups to your local dev environment. So it works best when you have a most-current MySQL backup as a symlink that updates nightly.
|
3
3
|
|
4
4
|
## Installation
|
5
5
|
|
6
6
|
`gem install db-rotator`
|
7
7
|
|
8
|
+
## Requirements
|
9
|
+
- This tool creates and drops databases, and runs any SQL in your dump (duh), so you have to give it a user that can do all of that. If you're using this on a local dev environment, it's recommended to setup user/pass in `~/.my.cnf` so that "mysql" works without `-u` or `-p`. You can configure DBRotator to work with any credentials, however. See config section.
|
10
|
+
- Disk space for N+1 database instances, where N is the amount you want to prune to (maximum DBs).
|
11
|
+
|
8
12
|
## Usage
|
9
13
|
|
10
|
-
|
11
|
-
|
12
|
-
|
14
|
+
**Minimal usage:**
|
15
|
+
|
16
|
+
Run: `db-rotator -p 'appdump_' -c 'scp db5:/opt/backups/latest.sql.bz2'`
|
17
|
+
|
18
|
+
**Or, from default config file** `~/.db-rotator.yml`:
|
19
|
+
|
20
|
+
```
|
21
|
+
db_prefix: "appdump_"
|
22
|
+
scp_command: "scp db5:/opt/backups/latest.sql.bz2"
|
23
|
+
```
|
24
|
+
|
25
|
+
Run: `db-rotator`
|
13
26
|
|
14
|
-
|
27
|
+
**Or, from a specific config file** `/whatever/rotator-config.conf`:
|
15
28
|
|
16
|
-
|
29
|
+
```
|
30
|
+
db_prefix: "appdump_"
|
31
|
+
scp_command: "scp db5:/opt/backups/latest.sql.bz2"
|
32
|
+
```
|
17
33
|
|
18
|
-
|
34
|
+
Run: `db-rotator -f /whatever/rotator-config.conf`
|
19
35
|
|
20
|
-
|
36
|
+
**Rotate nightly**, so you'll always have a fresh dump during your workday:
|
21
37
|
|
22
|
-
|
38
|
+
`0 3 * * * bash -lc "db-rotator -f /whatever/rotator-config.conf >> /some/log/file"`
|
23
39
|
|
24
|
-
|
25
|
-
db_prefix: "appdump_"
|
26
|
-
scp_command: "scp db5:/opt/backups/latest.sql.bz2"
|
27
|
-
```
|
40
|
+
## Configuration
|
28
41
|
|
29
|
-
|
42
|
+
Run `db-rotator` without any options to show config options.
|
30
43
|
|
31
|
-
|
44
|
+
### Required
|
45
|
+
#### db_prefix (-p)
|
46
|
+
Database naming prefix that will apply to all dumps rotated with DBRotator.
|
47
|
+
Example: `myproject_`, which might name a DB as myproject_09182013.
|
32
48
|
|
33
|
-
|
34
|
-
|
35
|
-
|
49
|
+
#### scp_command (-c)
|
50
|
+
How DBRotator retrieves your dumps. This ideally is an scp command, but really can be any command that receives a second argument of the dump destination.
|
51
|
+
Example: `scp hostname:/path/to/mysql/backups/backup_filename.sql.bz2`
|
36
52
|
|
37
|
-
|
53
|
+
### Optional
|
38
54
|
|
39
|
-
- **local_dump_destination** (-d). Where to put the dump, as a directory.
|
55
|
+
- **local_dump_destination** (-d). Where to put the dump, as a directory. The dump won't be deleted after running rotator. Default: `/tmp`
|
40
56
|
- **mysql_command** (-m). Used for all database management operations. Default: `mysql`
|
41
57
|
- **maximum_dbs** (-n). Maximum number of DBs to maintain, or null to disable pruning. Default: 2
|
42
58
|
- **unarchive_command** (-u). How to unarchive your dump to standard output. Default: `bzip2 -cd`
|
data/Rakefile
ADDED
data/bin/db-rotator
CHANGED
data/db-rotator.gemspec
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = "db-rotator"
|
3
|
-
s.version = "0.0.
|
3
|
+
s.version = "0.0.2"
|
4
4
|
s.summary = "Easy MySQL database rotation and pruning"
|
5
5
|
s.description = "Easy MySQL database rotation and pruning"
|
6
6
|
s.authors = ["Michael Nelson"]
|
7
7
|
s.email = "michael@nelsonware.com"
|
8
8
|
s.executables << "db-rotator"
|
9
9
|
s.files = `git ls-files`.split
|
10
|
-
|
10
|
+
s.test_files = `git ls-files -- test/*`.split
|
11
11
|
s.homepage = "https://github.com/mcnelson/db-rotator"
|
12
12
|
s.license = "LGPL-2.0"
|
13
13
|
|
data/lib/db_rotator.rb
CHANGED
@@ -63,7 +63,7 @@ class DBRotator
|
|
63
63
|
|
64
64
|
def grant_access
|
65
65
|
verbose_message "Granting ..."
|
66
|
-
mysql_exec "GRANT ALL ON #{todays_dbname}.* to
|
66
|
+
mysql_exec "GRANT ALL ON #{todays_dbname}.* to #{current_user}"
|
67
67
|
end
|
68
68
|
|
69
69
|
def populate_schemas
|
@@ -120,6 +120,10 @@ class DBRotator
|
|
120
120
|
mysql_exec "SHOW DATABASES"
|
121
121
|
end
|
122
122
|
|
123
|
+
def current_user
|
124
|
+
mysql_exec("SELECT current_user()").split("\n").last
|
125
|
+
end
|
126
|
+
|
123
127
|
def bash_exec(cmd, skip_raise = false)
|
124
128
|
out = `#{cmd} || echo '__failed'`
|
125
129
|
out = out.strip == "__failed" ? nil : out
|
data/lib/db_rotator_config.rb
CHANGED
@@ -8,26 +8,28 @@ class DBRotatorConfig
|
|
8
8
|
FILE = ".db-rotator.yml"
|
9
9
|
|
10
10
|
CONFIG = {
|
11
|
-
db_prefix: [['-p', "--db-prefix PREFIX", "Database naming prefix"], nil],
|
12
|
-
scp_command: [['-c', "--scp-command COMMAND", "Command to retrive dump. Receives second arg of dump path"], nil],
|
11
|
+
db_prefix: [['-p', "--db-prefix PREFIX", "Database naming prefix."], nil],
|
12
|
+
scp_command: [['-c', "--scp-command COMMAND", "Command to retrive dump. Receives second arg of dump path."], nil],
|
13
13
|
|
14
|
-
local_dump_destination: [['-d', "--local-destination [PATH]", "Where to put the dump, as a directory.
|
15
|
-
mysql_command: [['-m', "--mysql-command [COMMAND]", "Used for all database management operations.", "mysql"], "mysql"],
|
14
|
+
local_dump_destination: [['-d', "--local-destination [PATH]", "Where to put the dump, as a directory. The dump won't be deleted after running rotator.", "Default: /tmp"], "/tmp"],
|
15
|
+
mysql_command: [['-m', "--mysql-command [COMMAND]", "Used for all database management operations.", "Default: mysql"], "mysql"],
|
16
16
|
maximum_dbs: [['-n', "--maximum-dbs [N]", "Maximum number of DBs to maintain, or null to disable pruning.", "Default: 2"], 2],
|
17
17
|
unarchive_command: [['-u', "--unarchive-command [COMMAND]", "How to unarchive your dump to standard output.", "Default: bzip2 -cd"], "bzip2 -cd"],
|
18
18
|
unarchive_extra_pipe: [['-i', "--extra-pipes [SCRIPT1,SCRIPT2]", Array, "Any extra script(s) you want to run between unarchive & import.", "Example: -i 'script1,script2'"], nil],
|
19
|
-
reasonable_diskspace: [['-s', "--minimum-diskspace [GB]", "Rough estimate of temporary disk space required to import a typical dump, in GB."
|
19
|
+
reasonable_diskspace: [['-s', "--minimum-diskspace [GB]", "Rough estimate of temporary disk space required to import a typical dump, in GB."], nil],
|
20
20
|
rails_db_yaml_path: [['-y', "--rails-db-yaml [PATH]", "Updates database name in your YAML file when given.", "Default: nil"], nil],
|
21
|
-
rails_environments: [['-e', "--rails-db-environments [ENV1,ENV2]", Array, "In conjunction with -y, which rails envs to update DB name for.
|
21
|
+
rails_environments: [['-e', "--rails-db-environments [ENV1,ENV2]", Array, "In conjunction with -y, which rails envs to update DB name for. Default: development"], ["development"]],
|
22
22
|
|
23
|
-
config_file: [['-f', "--config-file PATH", "
|
23
|
+
config_file: [['-f', "--config-file PATH", "Runs rotator with configuration from this .yml file."], nil],
|
24
24
|
}
|
25
25
|
|
26
26
|
REQUIRED = %i(db_prefix scp_command)
|
27
|
+
EXCLUDE_FROM_GENERATE_FILE = %i(config_file dump_filename)
|
27
28
|
attr_reader :config
|
28
29
|
|
29
30
|
def initialize
|
30
31
|
@config = {}
|
32
|
+
@generate_output_file = nil
|
31
33
|
end
|
32
34
|
|
33
35
|
def configure
|
@@ -41,6 +43,12 @@ class DBRotatorConfig
|
|
41
43
|
check_required
|
42
44
|
add_default_values
|
43
45
|
add_derived_values
|
46
|
+
|
47
|
+
if @generate_output_file
|
48
|
+
generate_config
|
49
|
+
exit
|
50
|
+
end
|
51
|
+
|
44
52
|
rescue FileNotFoundError => e
|
45
53
|
puts "There was a problem loading configuration: #{e.message}"
|
46
54
|
puts cli_parser.summarize
|
@@ -78,6 +86,10 @@ class DBRotatorConfig
|
|
78
86
|
end
|
79
87
|
end
|
80
88
|
|
89
|
+
opts.on("-g", "--generate-config FILE", "Generates .yml config file from given arguments.") do |v|
|
90
|
+
@generate_output_file = v
|
91
|
+
end
|
92
|
+
|
81
93
|
opts.on("-v", "--verbose") do |v|
|
82
94
|
@config[:verbose] = v
|
83
95
|
end
|
@@ -97,6 +109,15 @@ class DBRotatorConfig
|
|
97
109
|
REQUIRED.each { |k| raise "config option '#{k.to_s} is required'" if @config[k].nil? }
|
98
110
|
end
|
99
111
|
|
112
|
+
def generate_config
|
113
|
+
File.write(@generate_output_file, config_yaml)
|
114
|
+
end
|
115
|
+
|
116
|
+
def config_yaml
|
117
|
+
require 'pry'; binding.pry
|
118
|
+
@config.reject { |k, _| EXCLUDE_FROM_GENERATE_FILE.include?(k) } .to_yaml
|
119
|
+
end
|
120
|
+
|
100
121
|
def add_default_values
|
101
122
|
CONFIG.each do |key, pair|
|
102
123
|
@config[key] ||= pair.last
|
@@ -105,7 +126,13 @@ class DBRotatorConfig
|
|
105
126
|
|
106
127
|
def add_derived_values
|
107
128
|
# Figure out the dump filename
|
108
|
-
|
129
|
+
cmd = if @config[:scp_command].strip.match(/^scp/)
|
130
|
+
@config[:scp_command].split(':').last
|
131
|
+
else
|
132
|
+
@config[:scp_command]
|
133
|
+
end
|
134
|
+
|
135
|
+
pn = Pathname.new(cmd)
|
109
136
|
@config[:dump_filename] = pn.basename.to_s
|
110
137
|
end
|
111
138
|
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
-- MySQL dump 10.13 Distrib 5.5.34, for Linux (x86_64)
|
2
|
+
--
|
3
|
+
-- Host: localhost Database: whatever
|
4
|
+
-- ------------------------------------------------------
|
5
|
+
-- Server version 5.5.34-32.0-log
|
6
|
+
|
7
|
+
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
|
8
|
+
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
|
9
|
+
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
|
10
|
+
/*!40101 SET NAMES utf8 */;
|
11
|
+
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
|
12
|
+
/*!40103 SET TIME_ZONE='+00:00' */;
|
13
|
+
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
|
14
|
+
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
|
15
|
+
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
|
16
|
+
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
|
17
|
+
|
18
|
+
--
|
19
|
+
-- Table structure for table `test`
|
20
|
+
--
|
21
|
+
|
22
|
+
DROP TABLE IF EXISTS `test`;
|
23
|
+
/*!40101 SET @saved_cs_client = @@character_set_client */;
|
24
|
+
/*!40101 SET character_set_client = utf8 */;
|
25
|
+
CREATE TABLE `test` (
|
26
|
+
`id` int(11) NOT NULL AUTO_INCREMENT,
|
27
|
+
`column_varchar` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,
|
28
|
+
`column_int` tinyint(1) DEFAULT NULL,
|
29
|
+
`column_date` datetime DEFAULT NULL,
|
30
|
+
`column_nonnull` varchar(45) COLLATE utf8_unicode_ci NOT NULL,
|
31
|
+
PRIMARY KEY (`id`)
|
32
|
+
) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
|
33
|
+
/*!40101 SET character_set_client = @saved_cs_client */;
|
34
|
+
|
35
|
+
--
|
36
|
+
-- Dumping data for table `test`
|
37
|
+
--
|
38
|
+
|
39
|
+
LOCK TABLES `test` WRITE;
|
40
|
+
/*!40000 ALTER TABLE `test` DISABLE KEYS */;
|
41
|
+
INSERT INTO `test` VALUES (1,'Herpa Derpa',1,'2013-01-01 01:10:20','Blargh'),(2,'Dorp Schlorp',0,'2013-09-18 04:50:12','Rawr');
|
42
|
+
/*!40000 ALTER TABLE `test` ENABLE KEYS */;
|
43
|
+
UNLOCK TABLES;
|
44
|
+
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
|
45
|
+
|
46
|
+
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
|
47
|
+
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
|
48
|
+
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
|
49
|
+
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
|
50
|
+
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
|
51
|
+
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
|
52
|
+
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
|
53
|
+
|
54
|
+
-- Dump completed on 2013-11-20 9:00:47
|
Binary file
|
Binary file
|
Binary file
|
data/test/helper.rb
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'minitest/autorun'
|
2
|
+
require_relative '../lib/db_rotator'
|
3
|
+
require_relative '../lib/db_rotator_config'
|
4
|
+
|
5
|
+
TEST_SCHEMA_PREFIX = "__minitest_"
|
6
|
+
|
7
|
+
def dummy_config(config={})
|
8
|
+
DBRotatorConfig.new.tap do |cfg|
|
9
|
+
cfg.instance_variable_set(:@config, {
|
10
|
+
db_prefix: TEST_SCHEMA_PREFIX,
|
11
|
+
scp_command: "cp test/fixtures/basic_dump.sql.bz2",
|
12
|
+
|
13
|
+
local_dump_destination: "/tmp"
|
14
|
+
}.merge(config))
|
15
|
+
|
16
|
+
cfg.add_default_values
|
17
|
+
cfg.add_derived_values
|
18
|
+
end
|
19
|
+
end
|
data/test/test_db_rotator.rb
CHANGED
@@ -1,6 +1,45 @@
|
|
1
|
-
|
2
|
-
require 'minitest'
|
1
|
+
require_relative '../test/helper'
|
3
2
|
|
4
3
|
describe DBRotator do
|
4
|
+
after do
|
5
|
+
`mysql -B -e 'SHOW SCHEMAS;'`.split.each do |schema|
|
6
|
+
if /^#{TEST_SCHEMA_PREFIX}/.match(schema)
|
7
|
+
`mysql -e "DROP SCHEMA #{schema}"`
|
8
|
+
end
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def dbname_with(time)
|
13
|
+
"__minitest_#{ time.strftime(DBRotator::TIME_FORMAT) }"
|
14
|
+
end
|
15
|
+
|
16
|
+
describe "integration test" do
|
17
|
+
it "rotates databases correctly" do
|
18
|
+
dbr = DBRotator.new(dummy_config)
|
19
|
+
|
20
|
+
t1 = Time.new(2013, 1, 1)
|
21
|
+
t2 = Time.new(2013, 1, 2)
|
22
|
+
t3 = Time.new(2013, 1, 3)
|
23
|
+
|
24
|
+
Time.stub(:now, t1) do
|
25
|
+
dbr.rotate
|
26
|
+
`mysql -B -e 'SHOW SCHEMAS;'`.must_include(dbname_with(t1))
|
27
|
+
`mysql -B -e 'SHOW TABLES FROM #{dbname_with(t1)};'`.must_include("test")
|
28
|
+
end
|
29
|
+
|
30
|
+
Time.stub(:now, t2) do
|
31
|
+
dbr.rotate
|
32
|
+
`mysql -B -e 'SHOW SCHEMAS;'`.must_include(dbname_with(t2))
|
33
|
+
`mysql -B -e 'SHOW SCHEMAS;'`.must_include(dbname_with(t1))
|
34
|
+
end
|
35
|
+
|
36
|
+
Time.stub(:now, t3) do
|
37
|
+
dbr.rotate
|
38
|
+
`mysql -B -e 'SHOW SCHEMAS;'`.must_include(dbname_with(t3))
|
39
|
+
`mysql -B -e 'SHOW SCHEMAS;'`.must_include(dbname_with(t2))
|
5
40
|
|
41
|
+
`mysql -B -e 'SHOW SCHEMAS;'`.wont_include(dbname_with(t1))
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
6
45
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: db-rotator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Nelson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-11-
|
11
|
+
date: 2013-11-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: minitest
|
@@ -34,10 +34,16 @@ files:
|
|
34
34
|
- .gitignore
|
35
35
|
- LICENSE
|
36
36
|
- README.md
|
37
|
+
- Rakefile
|
37
38
|
- bin/db-rotator
|
38
39
|
- db-rotator.gemspec
|
39
40
|
- lib/db_rotator.rb
|
40
41
|
- lib/db_rotator_config.rb
|
42
|
+
- test/fixtures/basic_dump.sql
|
43
|
+
- test/fixtures/basic_dump.sql.bz2
|
44
|
+
- test/fixtures/basic_dump.sql.gz
|
45
|
+
- test/fixtures/basic_dump.sql.zip
|
46
|
+
- test/helper.rb
|
41
47
|
- test/test_db_rotator.rb
|
42
48
|
homepage: https://github.com/mcnelson/db-rotator
|
43
49
|
licenses:
|
@@ -63,4 +69,10 @@ rubygems_version: 2.0.2
|
|
63
69
|
signing_key:
|
64
70
|
specification_version: 4
|
65
71
|
summary: Easy MySQL database rotation and pruning
|
66
|
-
test_files:
|
72
|
+
test_files:
|
73
|
+
- test/fixtures/basic_dump.sql
|
74
|
+
- test/fixtures/basic_dump.sql.bz2
|
75
|
+
- test/fixtures/basic_dump.sql.gz
|
76
|
+
- test/fixtures/basic_dump.sql.zip
|
77
|
+
- test/helper.rb
|
78
|
+
- test/test_db_rotator.rb
|