lhm 2.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -13
- data/.rubocop.yml +256 -0
- data/.travis.yml +2 -3
- data/CHANGELOG.md +21 -0
- data/README.md +30 -3
- data/Rakefile +2 -2
- data/bin/lhm-config.sh +7 -0
- data/bin/lhm-kill-queue +13 -15
- data/bin/lhm-spec-clobber.sh +1 -1
- data/bin/lhm-spec-grants.sh +2 -2
- data/bin/lhm-spec-setup-cluster.sh +1 -1
- data/gemfiles/ar-2.3_mysql.gemfile +1 -0
- data/lhm.gemspec +6 -6
- data/lib/lhm.rb +20 -8
- data/lib/lhm/atomic_switcher.rb +2 -1
- data/lib/lhm/chunker.rb +19 -19
- data/lib/lhm/command.rb +1 -1
- data/lib/lhm/connection.rb +12 -0
- data/lib/lhm/entangler.rb +5 -5
- data/lib/lhm/intersection.rb +29 -16
- data/lib/lhm/invoker.rb +3 -3
- data/lib/lhm/locked_switcher.rb +6 -6
- data/lib/lhm/migration.rb +5 -4
- data/lib/lhm/migrator.rb +39 -10
- data/lib/lhm/printer.rb +4 -6
- data/lib/lhm/sql_helper.rb +4 -4
- data/lib/lhm/table.rb +12 -12
- data/lib/lhm/version.rb +1 -1
- data/spec/fixtures/users.ddl +1 -1
- data/spec/integration/atomic_switcher_spec.rb +7 -7
- data/spec/integration/chunker_spec.rb +2 -2
- data/spec/integration/cleanup_spec.rb +34 -10
- data/spec/integration/entangler_spec.rb +11 -11
- data/spec/integration/integration_helper.rb +10 -3
- data/spec/integration/lhm_spec.rb +96 -46
- data/spec/integration/locked_switcher_spec.rb +7 -7
- data/spec/integration/table_spec.rb +15 -15
- data/spec/test_helper.rb +4 -4
- data/spec/unit/atomic_switcher_spec.rb +6 -6
- data/spec/unit/chunker_spec.rb +22 -9
- data/spec/unit/entangler_spec.rb +19 -19
- data/spec/unit/intersection_spec.rb +27 -15
- data/spec/unit/lhm_spec.rb +6 -6
- data/spec/unit/locked_switcher_spec.rb +14 -14
- data/spec/unit/migration_spec.rb +6 -6
- data/spec/unit/migrator_spec.rb +53 -41
- data/spec/unit/printer_spec.rb +7 -7
- data/spec/unit/sql_helper_spec.rb +10 -10
- data/spec/unit/table_spec.rb +11 -11
- data/spec/unit/throttler_spec.rb +12 -12
- metadata +12 -10
@@ -12,30 +12,30 @@ describe Lhm::LockedSwitcher do
|
|
12
12
|
|
13
13
|
before(:each) { connect_master! }
|
14
14
|
|
15
|
-
describe
|
15
|
+
describe 'switching' do
|
16
16
|
before(:each) do
|
17
|
-
@origin = table_create(
|
18
|
-
@destination = table_create(
|
17
|
+
@origin = table_create('origin')
|
18
|
+
@destination = table_create('destination')
|
19
19
|
@migration = Lhm::Migration.new(@origin, @destination)
|
20
20
|
end
|
21
21
|
|
22
|
-
it
|
22
|
+
it 'rename origin to archive' do
|
23
23
|
switcher = Lhm::LockedSwitcher.new(@migration, connection)
|
24
24
|
switcher.run
|
25
25
|
|
26
26
|
slave do
|
27
27
|
table_exists?(@origin).must_equal true
|
28
|
-
table_read(@migration.archive_name).columns.keys.must_include
|
28
|
+
table_read(@migration.archive_name).columns.keys.must_include 'origin'
|
29
29
|
end
|
30
30
|
end
|
31
31
|
|
32
|
-
it
|
32
|
+
it 'rename destination to origin' do
|
33
33
|
switcher = Lhm::LockedSwitcher.new(@migration, connection)
|
34
34
|
switcher.run
|
35
35
|
|
36
36
|
slave do
|
37
37
|
table_exists?(@destination).must_equal false
|
38
|
-
table_read(@origin.name).columns.keys.must_include
|
38
|
+
table_read(@origin.name).columns.keys.must_include 'destination'
|
39
39
|
end
|
40
40
|
end
|
41
41
|
end
|
@@ -8,38 +8,38 @@ describe Lhm::Table do
|
|
8
8
|
include IntegrationHelper
|
9
9
|
|
10
10
|
describe Lhm::Table::Parser do
|
11
|
-
describe
|
11
|
+
describe 'create table parsing' do
|
12
12
|
before(:each) do
|
13
13
|
connect_master!
|
14
14
|
@table = table_create(:users)
|
15
15
|
end
|
16
16
|
|
17
|
-
it
|
18
|
-
@table.name.must_equal(
|
17
|
+
it 'should parse table name in show create table' do
|
18
|
+
@table.name.must_equal('users')
|
19
19
|
end
|
20
20
|
|
21
|
-
it
|
22
|
-
@table.pk.must_equal(
|
21
|
+
it 'should parse primary key' do
|
22
|
+
@table.pk.must_equal('id')
|
23
23
|
end
|
24
24
|
|
25
|
-
it
|
26
|
-
@table.columns[
|
25
|
+
it 'should parse column type in show create table' do
|
26
|
+
@table.columns['username'][:type].must_equal('varchar(255)')
|
27
27
|
end
|
28
28
|
|
29
|
-
it
|
30
|
-
@table.columns[
|
29
|
+
it 'should parse column metadata' do
|
30
|
+
@table.columns['username'][:column_default].must_equal nil
|
31
31
|
end
|
32
32
|
|
33
|
-
it
|
33
|
+
it 'should parse indices' do
|
34
34
|
@table.
|
35
|
-
indices[
|
36
|
-
must_equal([
|
35
|
+
indices['index_users_on_username_and_created_at'].
|
36
|
+
must_equal(['username', 'created_at'])
|
37
37
|
end
|
38
38
|
|
39
|
-
it
|
39
|
+
it 'should parse index' do
|
40
40
|
@table.
|
41
|
-
indices[
|
42
|
-
must_equal([
|
41
|
+
indices['index_users_on_reference'].
|
42
|
+
must_equal(['reference'])
|
43
43
|
end
|
44
44
|
end
|
45
45
|
end
|
data/spec/test_helper.rb
CHANGED
@@ -4,12 +4,12 @@
|
|
4
4
|
require 'minitest/autorun'
|
5
5
|
require 'minitest/spec'
|
6
6
|
require 'minitest/mock'
|
7
|
-
require
|
8
|
-
require
|
7
|
+
require 'pathname'
|
8
|
+
require 'lhm'
|
9
9
|
|
10
10
|
$project = Pathname.new(File.dirname(__FILE__) + '/..').cleanpath
|
11
|
-
$spec = $project.join(
|
12
|
-
$fixtures = $spec.join(
|
11
|
+
$spec = $project.join('spec')
|
12
|
+
$fixtures = $spec.join('fixtures')
|
13
13
|
|
14
14
|
begin
|
15
15
|
require 'active_record'
|
@@ -12,19 +12,19 @@ describe Lhm::AtomicSwitcher do
|
|
12
12
|
|
13
13
|
before(:each) do
|
14
14
|
@start = Time.now
|
15
|
-
@origin = Lhm::Table.new(
|
16
|
-
@destination = Lhm::Table.new(
|
15
|
+
@origin = Lhm::Table.new('origin')
|
16
|
+
@destination = Lhm::Table.new('destination')
|
17
17
|
@migration = Lhm::Migration.new(@origin, @destination, @start)
|
18
18
|
@switcher = Lhm::AtomicSwitcher.new(@migration, nil)
|
19
19
|
end
|
20
20
|
|
21
|
-
describe
|
22
|
-
it
|
21
|
+
describe 'atomic switch' do
|
22
|
+
it 'should perform a single atomic rename' do
|
23
23
|
@switcher.
|
24
24
|
statements.
|
25
25
|
must_equal([
|
26
|
-
"rename table `origin` to `#{ @migration.archive_name }`, "
|
27
|
-
|
26
|
+
"rename table `origin` to `#{ @migration.archive_name }`, " \
|
27
|
+
'`destination` to `origin`'
|
28
28
|
])
|
29
29
|
end
|
30
30
|
end
|
data/spec/unit/chunker_spec.rb
CHANGED
@@ -12,14 +12,14 @@ describe Lhm::Chunker do
|
|
12
12
|
include UnitHelper
|
13
13
|
|
14
14
|
before(:each) do
|
15
|
-
@origin = Lhm::Table.new(
|
16
|
-
@destination = Lhm::Table.new(
|
15
|
+
@origin = Lhm::Table.new('foo')
|
16
|
+
@destination = Lhm::Table.new('bar')
|
17
17
|
@migration = Lhm::Migration.new(@origin, @destination)
|
18
18
|
@connection = MiniTest::Mock.new
|
19
19
|
# This is a poor man's stub
|
20
20
|
@throttler = Object.new
|
21
21
|
def @throttler.run
|
22
|
-
#noop
|
22
|
+
# noop
|
23
23
|
end
|
24
24
|
def @throttler.stride
|
25
25
|
1
|
@@ -29,8 +29,8 @@ describe Lhm::Chunker do
|
|
29
29
|
:limit => 10)
|
30
30
|
end
|
31
31
|
|
32
|
-
describe
|
33
|
-
it
|
32
|
+
describe '#run' do
|
33
|
+
it 'chunks the result set according to the stride size' do
|
34
34
|
def @throttler.stride
|
35
35
|
2
|
36
36
|
end
|
@@ -55,8 +55,8 @@ describe Lhm::Chunker do
|
|
55
55
|
@connection.verify
|
56
56
|
end
|
57
57
|
|
58
|
-
it
|
59
|
-
#roll our own stubbing
|
58
|
+
it 'handles stride changes during execution' do
|
59
|
+
# roll our own stubbing
|
60
60
|
def @throttler.stride
|
61
61
|
@run_count ||= 0
|
62
62
|
@run_count = @run_count + 1
|
@@ -84,7 +84,20 @@ describe Lhm::Chunker do
|
|
84
84
|
@connection.verify
|
85
85
|
end
|
86
86
|
|
87
|
-
it
|
87
|
+
it 'correctly copies single record tables' do
|
88
|
+
@chunker = Lhm::Chunker.new(@migration, @connection, :throttler => @throttler,
|
89
|
+
:start => 1,
|
90
|
+
:limit => 1)
|
91
|
+
|
92
|
+
@connection.expect(:update, 1) do |stmt|
|
93
|
+
stmt.first =~ /between 1 and 1/
|
94
|
+
end
|
95
|
+
|
96
|
+
@chunker.run
|
97
|
+
@connection.verify
|
98
|
+
end
|
99
|
+
|
100
|
+
it 'separates filter conditions from chunking conditions' do
|
88
101
|
@chunker = Lhm::Chunker.new(@migration, @connection, :throttler => @throttler,
|
89
102
|
:start => 1,
|
90
103
|
:limit => 2)
|
@@ -110,7 +123,7 @@ describe Lhm::Chunker do
|
|
110
123
|
end
|
111
124
|
|
112
125
|
def @migration.conditions
|
113
|
-
|
126
|
+
'inner join bar on foo.id = bar.foo_id'
|
114
127
|
end
|
115
128
|
|
116
129
|
@chunker.run
|
data/spec/unit/entangler_spec.rb
CHANGED
@@ -11,44 +11,44 @@ describe Lhm::Entangler do
|
|
11
11
|
include UnitHelper
|
12
12
|
|
13
13
|
before(:each) do
|
14
|
-
@origin = Lhm::Table.new(
|
15
|
-
@destination = Lhm::Table.new(
|
14
|
+
@origin = Lhm::Table.new('origin')
|
15
|
+
@destination = Lhm::Table.new('destination')
|
16
16
|
@migration = Lhm::Migration.new(@origin, @destination)
|
17
17
|
@entangler = Lhm::Entangler.new(@migration)
|
18
18
|
end
|
19
19
|
|
20
|
-
describe
|
20
|
+
describe 'activation' do
|
21
21
|
before(:each) do
|
22
|
-
@origin.columns[
|
23
|
-
@origin.columns[
|
22
|
+
@origin.columns['info'] = { :type => 'varchar(255)' }
|
23
|
+
@origin.columns['tags'] = { :type => 'varchar(255)' }
|
24
24
|
|
25
|
-
@destination.columns[
|
26
|
-
@destination.columns[
|
25
|
+
@destination.columns['info'] = { :type => 'varchar(255)' }
|
26
|
+
@destination.columns['tags'] = { :type => 'varchar(255)' }
|
27
27
|
end
|
28
28
|
|
29
|
-
it
|
29
|
+
it 'should create insert trigger to destination table' do
|
30
30
|
ddl = %Q{
|
31
31
|
create trigger `lhmt_ins_origin`
|
32
32
|
after insert on `origin` for each row
|
33
33
|
replace into `destination` (`info`, `tags`) /* large hadron migration */
|
34
|
-
values (NEW
|
34
|
+
values (`NEW`.`info`, `NEW`.`tags`)
|
35
35
|
}
|
36
36
|
|
37
37
|
@entangler.entangle.must_include strip(ddl)
|
38
38
|
end
|
39
39
|
|
40
|
-
it
|
40
|
+
it 'should create an update trigger to the destination table' do
|
41
41
|
ddl = %Q{
|
42
42
|
create trigger `lhmt_upd_origin`
|
43
43
|
after update on `origin` for each row
|
44
44
|
replace into `destination` (`info`, `tags`) /* large hadron migration */
|
45
|
-
values (NEW
|
45
|
+
values (`NEW`.`info`, `NEW`.`tags`)
|
46
46
|
}
|
47
47
|
|
48
48
|
@entangler.entangle.must_include strip(ddl)
|
49
49
|
end
|
50
50
|
|
51
|
-
it
|
51
|
+
it 'should create a delete trigger to the destination table' do
|
52
52
|
ddl = %Q{
|
53
53
|
create trigger `lhmt_del_origin`
|
54
54
|
after delete on `origin` for each row
|
@@ -60,17 +60,17 @@ describe Lhm::Entangler do
|
|
60
60
|
end
|
61
61
|
end
|
62
62
|
|
63
|
-
describe
|
64
|
-
it
|
65
|
-
@entangler.untangle.must_include(
|
63
|
+
describe 'removal' do
|
64
|
+
it 'should remove insert trigger' do
|
65
|
+
@entangler.untangle.must_include('drop trigger if exists `lhmt_ins_origin`')
|
66
66
|
end
|
67
67
|
|
68
|
-
it
|
69
|
-
@entangler.untangle.must_include(
|
68
|
+
it 'should remove update trigger' do
|
69
|
+
@entangler.untangle.must_include('drop trigger if exists `lhmt_upd_origin`')
|
70
70
|
end
|
71
71
|
|
72
|
-
it
|
73
|
-
@entangler.untangle.must_include(
|
72
|
+
it 'should remove delete trigger' do
|
73
|
+
@entangler.untangle.must_include('drop trigger if exists `lhmt_del_origin`')
|
74
74
|
end
|
75
75
|
end
|
76
76
|
end
|
@@ -9,31 +9,43 @@ require 'lhm/migrator'
|
|
9
9
|
describe Lhm::Intersection do
|
10
10
|
include UnitHelper
|
11
11
|
|
12
|
-
it
|
13
|
-
origin = Lhm::Table.new(
|
14
|
-
origin.columns[
|
15
|
-
origin.columns[
|
12
|
+
it 'should not have dropped changes' do
|
13
|
+
origin = Lhm::Table.new('origin')
|
14
|
+
origin.columns['dropped'] = varchar
|
15
|
+
origin.columns['retained'] = varchar
|
16
16
|
|
17
|
-
destination = Lhm::Table.new(
|
18
|
-
destination.columns[
|
17
|
+
destination = Lhm::Table.new('destination')
|
18
|
+
destination.columns['retained'] = varchar
|
19
19
|
|
20
20
|
intersection = Lhm::Intersection.new(origin, destination)
|
21
|
-
intersection.
|
21
|
+
intersection.destination.include?('dropped').must_equal(false)
|
22
22
|
end
|
23
23
|
|
24
|
-
it
|
25
|
-
origin = Lhm::Table.new(
|
26
|
-
origin.columns[
|
27
|
-
origin.columns[
|
24
|
+
it 'should have unchanged columns' do
|
25
|
+
origin = Lhm::Table.new('origin')
|
26
|
+
origin.columns['dropped'] = varchar
|
27
|
+
origin.columns['retained'] = varchar
|
28
28
|
|
29
|
-
destination = Lhm::Table.new(
|
30
|
-
destination.columns[
|
29
|
+
destination = Lhm::Table.new('destination')
|
30
|
+
destination.columns['retained'] = varchar
|
31
31
|
|
32
32
|
intersection = Lhm::Intersection.new(origin, destination)
|
33
|
-
intersection.
|
33
|
+
intersection.destination.must_equal(['retained'])
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'should have renamed columns' do
|
37
|
+
origin = Lhm::Table.new('origin')
|
38
|
+
origin.columns['old_name'] = varchar
|
39
|
+
|
40
|
+
destination = Lhm::Table.new('destination')
|
41
|
+
destination.columns['new_name'] = varchar
|
42
|
+
|
43
|
+
intersection = Lhm::Intersection.new(origin, destination, { 'old_name' => 'new_name' })
|
44
|
+
intersection.origin.must_equal(['old_name'])
|
45
|
+
intersection.destination.must_equal(['new_name'])
|
34
46
|
end
|
35
47
|
|
36
48
|
def varchar
|
37
|
-
{ :metadata =>
|
49
|
+
{ :metadata => 'VARCHAR(255)' }
|
38
50
|
end
|
39
51
|
end
|
data/spec/unit/lhm_spec.rb
CHANGED
@@ -8,22 +8,22 @@ describe Lhm do
|
|
8
8
|
Lhm.remove_class_variable :@@logger if Lhm.class_variable_defined? :@@logger
|
9
9
|
end
|
10
10
|
|
11
|
-
describe
|
11
|
+
describe 'logger' do
|
12
12
|
|
13
|
-
it
|
13
|
+
it 'should use the default parameters if no logger explicitly set' do
|
14
14
|
Lhm.logger.must_be_kind_of Logger
|
15
15
|
Lhm.logger.level.must_equal Logger::INFO
|
16
|
-
Lhm.logger.instance_eval{ @logdev }.dev.must_equal STDOUT
|
16
|
+
Lhm.logger.instance_eval { @logdev }.dev.must_equal STDOUT
|
17
17
|
end
|
18
18
|
|
19
|
-
it
|
19
|
+
it 'should use s new logger if set' do
|
20
20
|
l = Logger.new('omg.ponies')
|
21
21
|
l.level = Logger::ERROR
|
22
22
|
Lhm.logger = l
|
23
23
|
|
24
24
|
Lhm.logger.level.must_equal Logger::ERROR
|
25
|
-
Lhm.logger.instance_eval{ @logdev }.dev.must_be_kind_of File
|
26
|
-
Lhm.logger.instance_eval{ @logdev }.dev.path.must_equal 'omg.ponies'
|
25
|
+
Lhm.logger.instance_eval { @logdev }.dev.must_be_kind_of File
|
26
|
+
Lhm.logger.instance_eval { @logdev }.dev.path.must_equal 'omg.ponies'
|
27
27
|
end
|
28
28
|
end
|
29
29
|
end
|
@@ -12,39 +12,39 @@ describe Lhm::LockedSwitcher do
|
|
12
12
|
|
13
13
|
before(:each) do
|
14
14
|
@start = Time.now
|
15
|
-
@origin = Lhm::Table.new(
|
16
|
-
@destination = Lhm::Table.new(
|
15
|
+
@origin = Lhm::Table.new('origin')
|
16
|
+
@destination = Lhm::Table.new('destination')
|
17
17
|
@migration = Lhm::Migration.new(@origin, @destination, @start)
|
18
18
|
@switcher = Lhm::LockedSwitcher.new(@migration, nil)
|
19
19
|
end
|
20
20
|
|
21
|
-
describe
|
22
|
-
it
|
21
|
+
describe 'uncommitted' do
|
22
|
+
it 'should disable autocommit first' do
|
23
23
|
@switcher.
|
24
24
|
statements[0..1].
|
25
25
|
must_equal([
|
26
|
-
|
27
|
-
|
26
|
+
'set @lhm_auto_commit = @@session.autocommit',
|
27
|
+
'set session autocommit = 0'
|
28
28
|
])
|
29
29
|
end
|
30
30
|
|
31
|
-
it
|
31
|
+
it 'should reapply original autocommit settings at the end' do
|
32
32
|
@switcher.
|
33
33
|
statements[-1].
|
34
|
-
must_equal(
|
34
|
+
must_equal('set session autocommit = @lhm_auto_commit')
|
35
35
|
end
|
36
36
|
end
|
37
37
|
|
38
|
-
describe
|
39
|
-
it
|
38
|
+
describe 'switch' do
|
39
|
+
it 'should lock origin and destination table, switch, commit and unlock' do
|
40
40
|
@switcher.
|
41
41
|
switch.
|
42
42
|
must_equal([
|
43
|
-
|
43
|
+
'lock table `origin` write, `destination` write',
|
44
44
|
"alter table `origin` rename `#{ @migration.archive_name }`",
|
45
|
-
|
46
|
-
|
47
|
-
|
45
|
+
'alter table `destination` rename `origin`',
|
46
|
+
'commit',
|
47
|
+
'unlock tables'
|
48
48
|
])
|
49
49
|
end
|
50
50
|
end
|
data/spec/unit/migration_spec.rb
CHANGED
@@ -11,18 +11,18 @@ describe Lhm::Migration do
|
|
11
11
|
|
12
12
|
before(:each) do
|
13
13
|
@start = Time.now
|
14
|
-
@origin = Lhm::Table.new(
|
15
|
-
@destination = Lhm::Table.new(
|
14
|
+
@origin = Lhm::Table.new('origin')
|
15
|
+
@destination = Lhm::Table.new('destination')
|
16
16
|
@migration = Lhm::Migration.new(@origin, @destination, nil, @start)
|
17
17
|
end
|
18
18
|
|
19
|
-
it
|
20
|
-
stamp = "%Y_%m_%d_%H_%M_%S_#{
|
19
|
+
it 'should name archive' do
|
20
|
+
stamp = "%Y_%m_%d_%H_%M_%S_#{ '%03d' % (@start.usec / 1000) }"
|
21
21
|
@migration.archive_name.must_equal "lhma_#{ @start.strftime(stamp) }_origin"
|
22
22
|
end
|
23
23
|
|
24
|
-
it
|
25
|
-
migration = Lhm::Migration.new(OpenStruct.new(:name =>
|
24
|
+
it 'should limit table name to 64 characters' do
|
25
|
+
migration = Lhm::Migration.new(OpenStruct.new(:name => 'a_very_very_long_table_name_that_should_make_the_LHMA_table_go_over_64_chars'), nil)
|
26
26
|
migration.archive_name.size == 64
|
27
27
|
end
|
28
28
|
end
|