mysql-partitioner 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.rspec +2 -0
- data/.travis.yml +3 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +96 -0
- data/Rakefile +1 -0
- data/bin/console +14 -0
- data/bin/mysql-partitioner +8 -0
- data/bin/setup +7 -0
- data/lib/mysql/partitioner.rb +7 -0
- data/lib/mysql/partitioner/cli.rb +71 -0
- data/lib/mysql/partitioner/operation.rb +8 -0
- data/lib/mysql/partitioner/operation/abstract.rb +48 -0
- data/lib/mysql/partitioner/operation/range.rb +62 -0
- data/lib/mysql/partitioner/partition.rb +5 -0
- data/lib/mysql/partitioner/partition/range.rb +41 -0
- data/lib/mysql/partitioner/range.rb +63 -0
- data/lib/mysql/partitioner/session.rb +26 -0
- data/lib/mysql/partitioner/strategy.rb +16 -0
- data/lib/mysql/partitioner/strategy/partition_by_range_drop_by_time.rb +113 -0
- data/lib/mysql/partitioner/version.rb +5 -0
- data/mysql-partitioner.gemspec +29 -0
- data/sample/config.yaml +17 -0
- metadata +152 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: f2139ac031f639c559ada9a30ad71730ce15f83a
|
4
|
+
data.tar.gz: 76f06342030776eab6737e43575bf170c3f3be9a
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 60f3ee8d89d4e0377ee9a3d9d1b4a6c9dd8ba905d225f252295f51f841cc0cae18b53b80ecc36bcd0ddae95787ce2d59c93d67c8086ba599efa1bf3e68f18e5a
|
7
|
+
data.tar.gz: ae7b6b0c3845caa53cf6c519c6faa8f57de12c30bfb17039a4e8a7a528a975df93fb7e63de660b485eb8a44bd13e3c562a632e56eb74fada920ffc6c9b02206f
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2015 maedama
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
# Mysql::Partition
|
2
|
+
|
3
|
+
Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/mysql/partitioner`. To experiment with that code, run `bin/console` for an interactive prompt.
|
4
|
+
|
5
|
+
TODO: Delete this and the text above, and describe your gem
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
gem 'mysql-partitioner'
|
13
|
+
```
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install mysql-partitioner
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
`
|
26
|
+
Usage: mysql-partitioner [options]
|
27
|
+
-c, --config CONFIG_NAME
|
28
|
+
--cmd check or migrate
|
29
|
+
--dry-run
|
30
|
+
-d, --debug
|
31
|
+
`
|
32
|
+
|
33
|
+
|
34
|
+
## Example
|
35
|
+
|
36
|
+
`
|
37
|
+
CREATE TABLE `partition_test` (
|
38
|
+
`id` bigint(20) unsigned NOT NULL,
|
39
|
+
`created_at` datetime NOT NULL,
|
40
|
+
`updated_at` datetime NOT NULL,
|
41
|
+
PRIMARY KEY (`id`)
|
42
|
+
) Engine=InnoDB
|
43
|
+
`
|
44
|
+
|
45
|
+
Initialize first partition
|
46
|
+
|
47
|
+
```
|
48
|
+
% ./bin/mysql-partitioner -c sample/config.yaml --cmd migrate
|
49
|
+
I, [2015-10-22T12:23:55.415436 #10605] INFO -- : Running migrate on sample1 dry_run=false
|
50
|
+
I, [2015-10-22T12:23:55.423031 #10605] INFO -- : ALTER TABLE partition_test PARTITION BY RANGE (id) (
|
51
|
+
PARTITION p1000000 VALUES LESS THAN (1000000),
|
52
|
+
PARTITION p2000000 VALUES LESS THAN (2000000),
|
53
|
+
PARTITION p3000000 VALUES LESS THAN (3000000),
|
54
|
+
PARTITION p4000000 VALUES LESS THAN (4000000),
|
55
|
+
PARTITION p5000000 VALUES LESS THAN (5000000),
|
56
|
+
PARTITION pmax VALUES LESS THAN MAXVALUE)
|
57
|
+
|
58
|
+
I, [2015-10-22T12:23:55.524145 #10605] INFO -- : success
|
59
|
+
```
|
60
|
+
|
61
|
+
Insert old data
|
62
|
+
```
|
63
|
+
# Old
|
64
|
+
mysql> INSERT INTO partition_test VALUES(100, "2015-01-01 00:00:00", NOW());
|
65
|
+
Query OK, 1 row affected (0.01 sec)
|
66
|
+
|
67
|
+
# New
|
68
|
+
mysql> INSERT INTO partition_test VALUES(2000000, NOW(), NOW());
|
69
|
+
Query OK, 1 row affected (0.00 sec)
|
70
|
+
```
|
71
|
+
|
72
|
+
Update partition
|
73
|
+
```
|
74
|
+
% ./bin/mysql-partitioner -c sample/config.yaml --cmd migrate
|
75
|
+
I, [2015-10-22T12:27:02.860359 #10752] INFO -- : Running migrate on sample1 dry_run=false
|
76
|
+
I, [2015-10-22T12:27:02.878468 #10752] INFO -- : ALTER TABLE partition_test DROP PARTITION p1000000
|
77
|
+
I, [2015-10-22T12:27:03.022669 #10752] INFO -- : ALTER TABLE partition_test REORGANIZE PARTITION pmax INTO ( PARTITION p6000000 VALUES LESS THAN (6000000),
|
78
|
+
PARTITION p7000000 VALUES LESS THAN (7000000),
|
79
|
+
PARTITION p8000000 VALUES LESS THAN (8000000), PARTITION pmax VALUES LESS THAN MAXVALUE )
|
80
|
+
I, [2015-10-22T12:27:03.359712 #10752] INFO -- : success
|
81
|
+
```
|
82
|
+
|
83
|
+
|
84
|
+
## Development
|
85
|
+
|
86
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
|
87
|
+
|
88
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
89
|
+
|
90
|
+
## Contributing
|
91
|
+
|
92
|
+
1. Fork it ( https://github.com/[my-github-username]/mysql-partitioner/fork )
|
93
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
94
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
95
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
96
|
+
5. Create a new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "mysql/partitioner"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start
|
data/bin/setup
ADDED
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'logger'
|
3
|
+
require 'yaml'
|
4
|
+
require 'mysql2'
|
5
|
+
require 'mysql/partitioner/strategy'
|
6
|
+
require 'mysql/partitioner/session'
|
7
|
+
require 'deep_hash_transform'
|
8
|
+
|
9
|
+
module Mysql
|
10
|
+
module Partitioner
|
11
|
+
class Cli
|
12
|
+
def run()
|
13
|
+
|
14
|
+
options = {
|
15
|
+
:dry_run => false,
|
16
|
+
:debug => false,
|
17
|
+
:config => nil,
|
18
|
+
}
|
19
|
+
begin
|
20
|
+
ARGV.options do |opt|
|
21
|
+
opt.on("-c", "--config CONFIG_NAME") { |v| options[:config] = v }
|
22
|
+
opt.on("", "--cmd check or migrate") { |v| options[:cmd] = v }
|
23
|
+
opt.on('', '--dry-run') { |v| options[:dry_run] = v }
|
24
|
+
opt.on('-d', '--debug') { |v| options[:debug] = v }
|
25
|
+
opt.parse!
|
26
|
+
end
|
27
|
+
rescue => e
|
28
|
+
$stderr.puts e
|
29
|
+
exit 1
|
30
|
+
end
|
31
|
+
config = nil
|
32
|
+
begin
|
33
|
+
config = YAML.load_file(options[:config])
|
34
|
+
rescue => e
|
35
|
+
puts "Failed to load yaml file #{options[:config]} #{e}"
|
36
|
+
exit 1
|
37
|
+
end
|
38
|
+
|
39
|
+
logger = Logger.new(STDOUT)
|
40
|
+
logger.level = options[:debug] ? Logger::DEBUG : Logger::INFO
|
41
|
+
config.deep_symbolize_keys!
|
42
|
+
|
43
|
+
config.keys.each do|item|
|
44
|
+
task = config[item]
|
45
|
+
database = task[:database]
|
46
|
+
cli = Mysql2::Client.new(
|
47
|
+
:host => database[:host],
|
48
|
+
:username => database[:username],
|
49
|
+
:password => database[:password],
|
50
|
+
:port => database[:port],
|
51
|
+
:database => database[:name],
|
52
|
+
)
|
53
|
+
session = Mysql::Partitioner::Session.new(cli, options[:dry_run], logger)
|
54
|
+
strategy = Mysql::Partitioner::Strategy.build(session, task[:table], task[:strategy])
|
55
|
+
case options[:cmd]
|
56
|
+
when "check"
|
57
|
+
logger.info("Running check on #{item}")
|
58
|
+
strategy.check!
|
59
|
+
logger.info("success")
|
60
|
+
when "migrate"
|
61
|
+
logger.info("Running migrate on #{item} dry_run=#{options[:dry_run]}")
|
62
|
+
strategy.migrate
|
63
|
+
logger.info("success")
|
64
|
+
else
|
65
|
+
raise "Unknown command"
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
module Mysql
|
2
|
+
module Partitioner
|
3
|
+
module Operation
|
4
|
+
class Abstract
|
5
|
+
|
6
|
+
attr_reader :table, :session, :dry_run
|
7
|
+
|
8
|
+
def initialize(table, session)
|
9
|
+
@table = table
|
10
|
+
@session = session
|
11
|
+
end
|
12
|
+
|
13
|
+
def database
|
14
|
+
@database = @database || @session.query("SELECT DATABASE()").first.values[0] or raise "database not selected"
|
15
|
+
end
|
16
|
+
|
17
|
+
def get_partition_type()
|
18
|
+
results = @session.query(<<SQL)
|
19
|
+
SELECT PARTITION_EXPRESSION, PARTITION_DESCRIPTION, PARTITION_ORDINAL_POSITION, PARTITION_METHOD, SUBPARTITION_EXPRESSION
|
20
|
+
FROM INFORMATION_SCHEMA.PARTITIONS WHERE TABLE_NAME="#{ self.table }" AND TABLE_SCHEMA="#{ database }" LIMIT 1
|
21
|
+
SQL
|
22
|
+
row = results.first
|
23
|
+
if row.nil? then
|
24
|
+
raise "Table not found table=#{self.table} db=#{self.database}"
|
25
|
+
end
|
26
|
+
return row["PARTITION_METHOD"]
|
27
|
+
end
|
28
|
+
|
29
|
+
def partitionated?()
|
30
|
+
get_partition_type() != nil
|
31
|
+
end
|
32
|
+
|
33
|
+
def empty?()
|
34
|
+
@session.query("SELECT 1 FROM #{@table} LIMIT 1").first.nil?
|
35
|
+
end
|
36
|
+
|
37
|
+
def get_max_val(key)
|
38
|
+
@session.query("SELECT MAX(#{key}) FROM #{@table}").first.values[0]
|
39
|
+
end
|
40
|
+
|
41
|
+
def of_max_val(key, field, partition)
|
42
|
+
@session.query("SELECT MAX(#{key}), #{field} FROM #{@table} PARTITION(#{partition})").first.values[1]
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require 'mysql/partitioner/partition'
|
2
|
+
module Mysql
|
3
|
+
module Partitioner
|
4
|
+
module Operation
|
5
|
+
class Range < Abstract
|
6
|
+
|
7
|
+
def check!
|
8
|
+
type = get_partition_type()
|
9
|
+
if type != "RANGE" and type != nil
|
10
|
+
raise "Partition type mismatch #{type}"
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def get_current_bounded_partitions
|
15
|
+
get_current_partitions.select {| item| item.bounded? }
|
16
|
+
end
|
17
|
+
|
18
|
+
def get_current_partitions
|
19
|
+
results = self.session.query(<<SQL)
|
20
|
+
SELECT PARTITION_EXPRESSION, PARTITION_DESCRIPTION, PARTITION_ORDINAL_POSITION, PARTITION_METHOD, SUBPARTITION_EXPRESSION
|
21
|
+
FROM INFORMATION_SCHEMA.PARTITIONS WHERE TABLE_NAME="#{ self.table }" AND TABLE_SCHEMA="#{ self.database }"
|
22
|
+
SQL
|
23
|
+
results.map {|item|
|
24
|
+
Mysql::Partitioner::Partition::Range.new(item["PARTITION_DESCRIPTION"])
|
25
|
+
}
|
26
|
+
end
|
27
|
+
|
28
|
+
def add_partitions(partitions)
|
29
|
+
return if partitions.empty?
|
30
|
+
partition_defs = partitions.select.map {|item|
|
31
|
+
item.to_partition_def
|
32
|
+
}.join(",\n ")
|
33
|
+
self.session.alter("ALTER TABLE #{self.table} REORGANIZE PARTITION pmax INTO ( #{partition_defs }, PARTITION pmax VALUES LESS THAN MAXVALUE )" )
|
34
|
+
end
|
35
|
+
|
36
|
+
def create_partitions(key, partitions)
|
37
|
+
|
38
|
+
partition_defs = partitions.select.map {|item|
|
39
|
+
item.to_partition_def
|
40
|
+
}.join(",\n ")
|
41
|
+
self.session.alter(<<SQL)
|
42
|
+
ALTER TABLE #{self.table} PARTITION BY RANGE (#{key}) (
|
43
|
+
#{partition_defs},
|
44
|
+
PARTITION pmax VALUES LESS THAN MAXVALUE)
|
45
|
+
SQL
|
46
|
+
|
47
|
+
end
|
48
|
+
|
49
|
+
def drop_partitions(partitions)
|
50
|
+
return if partitions.empty?
|
51
|
+
names = partitions.map{|item| item.name }.join(",")
|
52
|
+
self.session.alter("ALTER TABLE #{self.table} DROP PARTITION #{names}" )
|
53
|
+
end
|
54
|
+
|
55
|
+
def migrate_partitions(old_partitions, new_partitions)
|
56
|
+
self.drop_partitions(old_partitions - new_partitions)
|
57
|
+
self.add_partitions(new_partitions - old_partitions)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module Mysql
|
2
|
+
module Partitioner
|
3
|
+
module Partition
|
4
|
+
class Range
|
5
|
+
attr_accessor :name, :less_than
|
6
|
+
|
7
|
+
def initialize(less_than, name=nil)
|
8
|
+
self.less_than = less_than == "MAXVALUE" ? Float::MAX : less_than.to_i
|
9
|
+
if name == nil then
|
10
|
+
name = self.build_name
|
11
|
+
end
|
12
|
+
self.name = name
|
13
|
+
end
|
14
|
+
|
15
|
+
def bounded?
|
16
|
+
self.less_than < Float::MAX
|
17
|
+
end
|
18
|
+
|
19
|
+
def build_name()
|
20
|
+
"p" + self.less_than.to_s
|
21
|
+
end
|
22
|
+
|
23
|
+
def to_partition_def
|
24
|
+
"PARTITION #{self.name} VALUES LESS THAN (#{self.less_than})"
|
25
|
+
end
|
26
|
+
|
27
|
+
def eql?(other)
|
28
|
+
self == other
|
29
|
+
end
|
30
|
+
|
31
|
+
def ==(other)
|
32
|
+
other.instance_of?(self.class) && other.name == self.name
|
33
|
+
end
|
34
|
+
|
35
|
+
def hash
|
36
|
+
self.name.hash
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
module Mysql
|
2
|
+
module Partition
|
3
|
+
class Range
|
4
|
+
|
5
|
+
def initialize(cli, partition)
|
6
|
+
@client = cli
|
7
|
+
@partition = partition
|
8
|
+
load_partitions
|
9
|
+
end
|
10
|
+
|
11
|
+
def load_partitions()
|
12
|
+
@database = @client.query("SELECT DATABASE()").first.values[0]
|
13
|
+
results = @client.query(<<SQL)
|
14
|
+
SELECT PARTITION_EXPRESSION, PARTITION_DESCRIPTION, PARTITION_ORDINAL_POSITION, PARTITION_METHOD, SUBPARTITION_EXPRESSION
|
15
|
+
FROM INFORMATION_SCHEMA.PARTITIONS WHERE TABLE_NAME="#{@partition["table"]}" AND TABLE_SCHEMA="#{@database}"
|
16
|
+
SQL
|
17
|
+
row = results.first
|
18
|
+
if row.nil? then
|
19
|
+
raise "Table schema not found"
|
20
|
+
end
|
21
|
+
|
22
|
+
if row["PARTITION_METHOD"] != "RANGE" and !row["PARTITION_METHOD"].nil? then
|
23
|
+
raise "Not a range partition"
|
24
|
+
end
|
25
|
+
|
26
|
+
@partitions = results.select{|item| !item[:PARTITION_METHOD].nil? }.map {|item|
|
27
|
+
{ name: item[:partition_description], less_than: item[:PARTITION_ORDINAL_POSITION] }
|
28
|
+
}
|
29
|
+
end
|
30
|
+
|
31
|
+
def get_next_partitions()
|
32
|
+
next_partition = @partitions.dup
|
33
|
+
next_partition = next_partition - find_old_partitions()
|
34
|
+
next_partition = next_partition + build_new_partitions()
|
35
|
+
end
|
36
|
+
|
37
|
+
def find_old_partitions()
|
38
|
+
@partitions.select {|item|
|
39
|
+
find_most_recent_item(item) <= config[:target_date]
|
40
|
+
}
|
41
|
+
end
|
42
|
+
|
43
|
+
def build_new_partitions()
|
44
|
+
max_val = select_max_val()
|
45
|
+
partitioned_upto = @partitions.last ? @partitions.last["less_than"] : 0
|
46
|
+
open_partitions = @partitions.select { |item| item[:less_than] > max_val }.size
|
47
|
+
buildable = @partition["prepared"] - open_partitions
|
48
|
+
result = []
|
49
|
+
for i in (1 .. buildable) do
|
50
|
+
new_less_than = partitioned_upto + @partition["interval"]
|
51
|
+
result.push({ less_than: new_less_than, name: "p" + new_less_than.to_s })
|
52
|
+
partitioned_upto = new_less_than
|
53
|
+
end
|
54
|
+
result
|
55
|
+
end
|
56
|
+
|
57
|
+
def select_max_val()
|
58
|
+
@client.query("SELECT MAX(#{@partition["key"]}) FROM #{@partition["table"]}").first.values[0] or 0
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module Mysql
|
2
|
+
module Partitioner
|
3
|
+
class Session
|
4
|
+
|
5
|
+
def initialize(client, dry_run, logger)
|
6
|
+
@client = client
|
7
|
+
@dry_run = dry_run
|
8
|
+
@alters = []
|
9
|
+
@logger = logger
|
10
|
+
end
|
11
|
+
|
12
|
+
def query(query)
|
13
|
+
raise "Use do_alter for alter query" if query.match(/ALTER/i)
|
14
|
+
@logger.debug(query)
|
15
|
+
@client.query(query)
|
16
|
+
end
|
17
|
+
|
18
|
+
def alter(query)
|
19
|
+
@alters.push(query)
|
20
|
+
@logger.info(query)
|
21
|
+
@client.query(query) if @dry_run == false
|
22
|
+
true
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'mysql/partitioner/strategy/partition_by_range_drop_by_time'
|
2
|
+
require 'mysql/partitioner/operation'
|
3
|
+
|
4
|
+
module Mysql
|
5
|
+
module Partitioner
|
6
|
+
module Strategy
|
7
|
+
def self.build(session, table, config)
|
8
|
+
case config[:name]
|
9
|
+
when "partition_by_range_drop_by_time" then
|
10
|
+
PartitionByRangeDropByTime.new(Mysql::Partitioner::Operation::Range.new(table, session), config)
|
11
|
+
else raise "Unknown strategy error"
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
require 'mysql/partitioner/partition'
|
2
|
+
module Mysql
|
3
|
+
module Partitioner
|
4
|
+
module Strategy
|
5
|
+
class PartitionByRangeDropByTime
|
6
|
+
def initialize(operation, config)
|
7
|
+
@operation = operation or raise "operation not specified"
|
8
|
+
@key = config[:key] or raise "Key not specified"
|
9
|
+
@time_key = config[:time_key] or raise "time column not specified"
|
10
|
+
@ttl = config[:ttl] or raise "ttl not specified"
|
11
|
+
@range = config[:range] or raise "range not specified"
|
12
|
+
@min_empty_partitions = config[:min_empty_partitions] or raise "min_empty_partitions not specified"
|
13
|
+
@desirable_empty_partitions = config[:desirable_empty_partitions] or raise "desirable_empty_partitions not specified"
|
14
|
+
@operation.check!
|
15
|
+
end
|
16
|
+
|
17
|
+
def check!()
|
18
|
+
|
19
|
+
raise "Not partitioned" unless @operation.partitionated?
|
20
|
+
current = @operation.get_current_bounded_partitions
|
21
|
+
|
22
|
+
empty = find_empty_partitions(current)
|
23
|
+
if empty.size < @min_empty_partitions then
|
24
|
+
raise "empty partitions is less than minimum was=#{empty.size} should_be=#{@min_empty_partitions}"
|
25
|
+
else
|
26
|
+
true
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def check()
|
31
|
+
success = true
|
32
|
+
begin
|
33
|
+
check!
|
34
|
+
rescue => e
|
35
|
+
success = false
|
36
|
+
end
|
37
|
+
return success
|
38
|
+
end
|
39
|
+
|
40
|
+
def migrate()
|
41
|
+
if @operation.partitionated? then
|
42
|
+
update_partitions()
|
43
|
+
else
|
44
|
+
initialize_partitions()
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def initialize_partitions()
|
49
|
+
max_less_than = 0
|
50
|
+
new = []
|
51
|
+
@desirable_empty_partitions.times do
|
52
|
+
max_less_than = max_less_than + @range
|
53
|
+
new.push( Mysql::Partitioner::Partition::Range.new(max_less_than) )
|
54
|
+
end
|
55
|
+
@operation.create_partitions(@key, new)
|
56
|
+
end
|
57
|
+
|
58
|
+
def update_partitions()
|
59
|
+
current = @operation.get_current_bounded_partitions
|
60
|
+
raise "partition is some how empty" if current.empty?
|
61
|
+
empty = find_empty_partitions(current)
|
62
|
+
|
63
|
+
new = []
|
64
|
+
if empty.size < @desirable_empty_partitions then
|
65
|
+
max_less_than = current.last.less_than
|
66
|
+
( @desirable_empty_partitions - empty.size ).times do
|
67
|
+
max_less_than = max_less_than + @range
|
68
|
+
new.push( Mysql::Partitioner::Partition::Range.new(max_less_than) )
|
69
|
+
end
|
70
|
+
end
|
71
|
+
old = find_old_partitions(current)
|
72
|
+
@operation.migrate_partitions(current, current - old + new)
|
73
|
+
end
|
74
|
+
|
75
|
+
|
76
|
+
def find_empty_partitions(current_partitions)
|
77
|
+
|
78
|
+
max_val = @operation.get_max_val(@key)
|
79
|
+
return current_partitions if max_val.nil?
|
80
|
+
max_val = max_val.to_i
|
81
|
+
max_active_partition = find_partition(current_partitions, max_val)
|
82
|
+
raise "partition not found error" if max_active_partition.nil?
|
83
|
+
|
84
|
+
current_partitions.select{|item|
|
85
|
+
item.less_than > max_val && item != max_active_partition
|
86
|
+
}
|
87
|
+
end
|
88
|
+
|
89
|
+
def find_old_partitions(current_partitions)
|
90
|
+
|
91
|
+
max_val = @operation.get_max_val(@key)
|
92
|
+
return [] if max_val.nil?
|
93
|
+
max_val = max_val.to_i
|
94
|
+
|
95
|
+
max_active_partition = find_partition(current_partitions, max_val)
|
96
|
+
raise "partition not found errorr" if max_active_partition.nil?
|
97
|
+
|
98
|
+
old_index = current_partitions.rindex{|item|
|
99
|
+
time = @operation.of_max_val(@key, @time_key, item.name)
|
100
|
+
time && time.to_i + @ttl < Time.now.to_i && item != max_active_partition
|
101
|
+
}
|
102
|
+
|
103
|
+
|
104
|
+
old_index && old_index >= 0 ? current_partitions[0 .. old_index] : []
|
105
|
+
end
|
106
|
+
|
107
|
+
def find_partition(partitions, val)
|
108
|
+
return partitions.find{|item| item.less_than > val}
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'mysql/partitioner/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "mysql-partitioner"
|
8
|
+
spec.version = Mysql::Partitioner::VERSION
|
9
|
+
spec.authors = ["maedama"]
|
10
|
+
spec.email = ["maedama85@gmail.com"]
|
11
|
+
|
12
|
+
spec.summary = %q{mysql partition management tools}
|
13
|
+
spec.description = %q{mysql partition management ttools}
|
14
|
+
spec.homepage = "https://github.com/maedama/mysql-partitioner"
|
15
|
+
spec.license = "MIT"
|
16
|
+
|
17
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
18
|
+
spec.bindir = "exe"
|
19
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
20
|
+
spec.require_paths = ["lib"]
|
21
|
+
|
22
|
+
spec.add_dependency "mysql2"
|
23
|
+
spec.add_dependency "deep_hash_transform"
|
24
|
+
|
25
|
+
spec.add_development_dependency "rspec", ">= 3.0.0"
|
26
|
+
spec.add_development_dependency "rspec-instafail"
|
27
|
+
spec.add_development_dependency "bundler", "~> 1.9"
|
28
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
29
|
+
end
|
data/sample/config.yaml
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
sample1:
|
2
|
+
database:
|
3
|
+
name: test
|
4
|
+
host: 127.0.0.1
|
5
|
+
username: root
|
6
|
+
password:
|
7
|
+
table: partition_test
|
8
|
+
strategy:
|
9
|
+
name: partition_by_range_drop_by_time
|
10
|
+
key: id
|
11
|
+
time_key: created_at
|
12
|
+
range: 1000000
|
13
|
+
min_empty_partitions: 3
|
14
|
+
desirable_empty_partitions: 5
|
15
|
+
ttl: 864000
|
16
|
+
|
17
|
+
|
metadata
ADDED
@@ -0,0 +1,152 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: mysql-partitioner
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- maedama
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-10-22 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: mysql2
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: deep_hash_transform
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 3.0.0
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 3.0.0
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rspec-instafail
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: bundler
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '1.9'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '1.9'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rake
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '10.0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '10.0'
|
97
|
+
description: mysql partition management ttools
|
98
|
+
email:
|
99
|
+
- maedama85@gmail.com
|
100
|
+
executables: []
|
101
|
+
extensions: []
|
102
|
+
extra_rdoc_files: []
|
103
|
+
files:
|
104
|
+
- ".gitignore"
|
105
|
+
- ".rspec"
|
106
|
+
- ".travis.yml"
|
107
|
+
- Gemfile
|
108
|
+
- LICENSE.txt
|
109
|
+
- README.md
|
110
|
+
- Rakefile
|
111
|
+
- bin/console
|
112
|
+
- bin/mysql-partitioner
|
113
|
+
- bin/setup
|
114
|
+
- lib/mysql/partitioner.rb
|
115
|
+
- lib/mysql/partitioner/cli.rb
|
116
|
+
- lib/mysql/partitioner/operation.rb
|
117
|
+
- lib/mysql/partitioner/operation/abstract.rb
|
118
|
+
- lib/mysql/partitioner/operation/range.rb
|
119
|
+
- lib/mysql/partitioner/partition.rb
|
120
|
+
- lib/mysql/partitioner/partition/range.rb
|
121
|
+
- lib/mysql/partitioner/range.rb
|
122
|
+
- lib/mysql/partitioner/session.rb
|
123
|
+
- lib/mysql/partitioner/strategy.rb
|
124
|
+
- lib/mysql/partitioner/strategy/partition_by_range_drop_by_time.rb
|
125
|
+
- lib/mysql/partitioner/version.rb
|
126
|
+
- mysql-partitioner.gemspec
|
127
|
+
- sample/config.yaml
|
128
|
+
homepage: https://github.com/maedama/mysql-partitioner
|
129
|
+
licenses:
|
130
|
+
- MIT
|
131
|
+
metadata: {}
|
132
|
+
post_install_message:
|
133
|
+
rdoc_options: []
|
134
|
+
require_paths:
|
135
|
+
- lib
|
136
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
137
|
+
requirements:
|
138
|
+
- - ">="
|
139
|
+
- !ruby/object:Gem::Version
|
140
|
+
version: '0'
|
141
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - ">="
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '0'
|
146
|
+
requirements: []
|
147
|
+
rubyforge_project:
|
148
|
+
rubygems_version: 2.4.5
|
149
|
+
signing_key:
|
150
|
+
specification_version: 4
|
151
|
+
summary: mysql partition management tools
|
152
|
+
test_files: []
|