lhm-shopify 3.3.5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (94) hide show
  1. checksums.yaml +7 -0
  2. data/.github/workflows/test.yml +34 -0
  3. data/.gitignore +17 -0
  4. data/.rubocop.yml +183 -0
  5. data/.travis.yml +21 -0
  6. data/CHANGELOG.md +216 -0
  7. data/Gemfile +5 -0
  8. data/LICENSE +27 -0
  9. data/README.md +284 -0
  10. data/Rakefile +22 -0
  11. data/bin/.gitkeep +0 -0
  12. data/dbdeployer/config.json +32 -0
  13. data/dbdeployer/install.sh +64 -0
  14. data/dev.yml +20 -0
  15. data/gemfiles/ar-2.3_mysql.gemfile +6 -0
  16. data/gemfiles/ar-3.2_mysql.gemfile +5 -0
  17. data/gemfiles/ar-3.2_mysql2.gemfile +5 -0
  18. data/gemfiles/ar-4.0_mysql2.gemfile +5 -0
  19. data/gemfiles/ar-4.1_mysql2.gemfile +5 -0
  20. data/gemfiles/ar-4.2_mysql2.gemfile +5 -0
  21. data/gemfiles/ar-5.0_mysql2.gemfile +5 -0
  22. data/lhm.gemspec +34 -0
  23. data/lib/lhm.rb +131 -0
  24. data/lib/lhm/atomic_switcher.rb +52 -0
  25. data/lib/lhm/chunk_finder.rb +32 -0
  26. data/lib/lhm/chunk_insert.rb +51 -0
  27. data/lib/lhm/chunker.rb +87 -0
  28. data/lib/lhm/cleanup/current.rb +74 -0
  29. data/lib/lhm/command.rb +48 -0
  30. data/lib/lhm/entangler.rb +117 -0
  31. data/lib/lhm/intersection.rb +51 -0
  32. data/lib/lhm/invoker.rb +98 -0
  33. data/lib/lhm/locked_switcher.rb +74 -0
  34. data/lib/lhm/migration.rb +43 -0
  35. data/lib/lhm/migrator.rb +237 -0
  36. data/lib/lhm/printer.rb +59 -0
  37. data/lib/lhm/railtie.rb +9 -0
  38. data/lib/lhm/sql_helper.rb +77 -0
  39. data/lib/lhm/sql_retry.rb +61 -0
  40. data/lib/lhm/table.rb +121 -0
  41. data/lib/lhm/table_name.rb +23 -0
  42. data/lib/lhm/test_support.rb +35 -0
  43. data/lib/lhm/throttler.rb +36 -0
  44. data/lib/lhm/throttler/slave_lag.rb +145 -0
  45. data/lib/lhm/throttler/threads_running.rb +53 -0
  46. data/lib/lhm/throttler/time.rb +29 -0
  47. data/lib/lhm/timestamp.rb +11 -0
  48. data/lib/lhm/version.rb +6 -0
  49. data/shipit.rubygems.yml +0 -0
  50. data/spec/.lhm.example +4 -0
  51. data/spec/README.md +58 -0
  52. data/spec/fixtures/bigint_table.ddl +4 -0
  53. data/spec/fixtures/composite_primary_key.ddl +7 -0
  54. data/spec/fixtures/custom_primary_key.ddl +6 -0
  55. data/spec/fixtures/destination.ddl +6 -0
  56. data/spec/fixtures/lines.ddl +7 -0
  57. data/spec/fixtures/origin.ddl +6 -0
  58. data/spec/fixtures/permissions.ddl +5 -0
  59. data/spec/fixtures/small_table.ddl +4 -0
  60. data/spec/fixtures/tracks.ddl +5 -0
  61. data/spec/fixtures/users.ddl +14 -0
  62. data/spec/fixtures/wo_id_int_column.ddl +6 -0
  63. data/spec/integration/atomic_switcher_spec.rb +93 -0
  64. data/spec/integration/chunk_insert_spec.rb +29 -0
  65. data/spec/integration/chunker_spec.rb +185 -0
  66. data/spec/integration/cleanup_spec.rb +136 -0
  67. data/spec/integration/entangler_spec.rb +66 -0
  68. data/spec/integration/integration_helper.rb +237 -0
  69. data/spec/integration/invoker_spec.rb +33 -0
  70. data/spec/integration/lhm_spec.rb +585 -0
  71. data/spec/integration/lock_wait_timeout_spec.rb +30 -0
  72. data/spec/integration/locked_switcher_spec.rb +50 -0
  73. data/spec/integration/sql_retry/lock_wait_spec.rb +125 -0
  74. data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +101 -0
  75. data/spec/integration/table_spec.rb +91 -0
  76. data/spec/test_helper.rb +32 -0
  77. data/spec/unit/atomic_switcher_spec.rb +31 -0
  78. data/spec/unit/chunk_finder_spec.rb +73 -0
  79. data/spec/unit/chunk_insert_spec.rb +44 -0
  80. data/spec/unit/chunker_spec.rb +166 -0
  81. data/spec/unit/entangler_spec.rb +124 -0
  82. data/spec/unit/intersection_spec.rb +51 -0
  83. data/spec/unit/lhm_spec.rb +29 -0
  84. data/spec/unit/locked_switcher_spec.rb +51 -0
  85. data/spec/unit/migrator_spec.rb +146 -0
  86. data/spec/unit/printer_spec.rb +97 -0
  87. data/spec/unit/sql_helper_spec.rb +32 -0
  88. data/spec/unit/table_name_spec.rb +39 -0
  89. data/spec/unit/table_spec.rb +47 -0
  90. data/spec/unit/throttler/slave_lag_spec.rb +317 -0
  91. data/spec/unit/throttler/threads_running_spec.rb +64 -0
  92. data/spec/unit/throttler_spec.rb +124 -0
  93. data/spec/unit/unit_helper.rb +13 -0
  94. metadata +239 -0
@@ -0,0 +1,29 @@
1
+ # Copyright (c) 2011 - 2013, SoundCloud Ltd.
2
+
3
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
4
+
5
+ describe Lhm do
6
+
7
+ before(:each) do
8
+ Lhm.remove_class_variable :@@logger if Lhm.class_variable_defined? :@@logger
9
+ end
10
+
11
+ describe 'logger' do
12
+
13
+ it 'should use the default parameters if no logger explicitly set' do
14
+ Lhm.logger.must_be_kind_of Logger
15
+ Lhm.logger.level.must_equal Logger::INFO
16
+ Lhm.logger.instance_eval { @logdev }.dev.must_equal STDOUT
17
+ end
18
+
19
+ it 'should use s new logger if set' do
20
+ l = Logger.new('omg.ponies')
21
+ l.level = Logger::ERROR
22
+ Lhm.logger = l
23
+
24
+ Lhm.logger.level.must_equal Logger::ERROR
25
+ Lhm.logger.instance_eval { @logdev }.dev.must_be_kind_of File
26
+ Lhm.logger.instance_eval { @logdev }.dev.path.must_equal 'omg.ponies'
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,51 @@
1
+ # Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
5
+
6
+ require 'lhm/table'
7
+ require 'lhm/migration'
8
+ require 'lhm/locked_switcher'
9
+
10
+ describe Lhm::LockedSwitcher do
11
+ include UnitHelper
12
+
13
+ before(:each) do
14
+ @start = Time.now
15
+ @origin = Lhm::Table.new('origin')
16
+ @destination = Lhm::Table.new('destination')
17
+ @migration = Lhm::Migration.new(@origin, @destination, @start)
18
+ @switcher = Lhm::LockedSwitcher.new(@migration, nil)
19
+ end
20
+
21
+ describe 'uncommitted' do
22
+ it 'should disable autocommit first' do
23
+ @switcher.
24
+ statements[0..1].
25
+ must_equal([
26
+ 'set @lhm_auto_commit = @@session.autocommit',
27
+ 'set session autocommit = 0'
28
+ ])
29
+ end
30
+
31
+ it 'should reapply original autocommit settings at the end' do
32
+ @switcher.
33
+ statements[-1].
34
+ must_equal('set session autocommit = @lhm_auto_commit')
35
+ end
36
+ end
37
+
38
+ describe 'switch' do
39
+ it 'should lock origin and destination table, switch, commit and unlock' do
40
+ @switcher.
41
+ switch.
42
+ must_equal([
43
+ 'lock table `origin` write, `destination` write',
44
+ "alter table `origin` rename `#{ @migration.archive_name }`",
45
+ 'alter table `destination` rename `origin`',
46
+ 'commit',
47
+ 'unlock tables'
48
+ ])
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,146 @@
1
+ # Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
5
+
6
+ require 'lhm/table'
7
+ require 'lhm/migrator'
8
+
9
+ describe Lhm::Migrator do
10
+ include UnitHelper
11
+
12
+ before(:each) do
13
+ @table = Lhm::Table.new('alt')
14
+ @creator = Lhm::Migrator.new(@table)
15
+ end
16
+
17
+ describe 'index changes' do
18
+ it 'should add an index' do
19
+ @creator.add_index(:a)
20
+
21
+ @creator.statements.must_equal([
22
+ 'create index `index_alt_on_a` on `lhmn_alt` (`a`)'
23
+ ])
24
+ end
25
+
26
+ it 'should add a composite index' do
27
+ @creator.add_index([:a, :b])
28
+
29
+ @creator.statements.must_equal([
30
+ 'create index `index_alt_on_a_and_b` on `lhmn_alt` (`a`, `b`)'
31
+ ])
32
+ end
33
+
34
+ it 'should add an index with prefix length' do
35
+ @creator.add_index(['a(10)', 'b'])
36
+
37
+ @creator.statements.must_equal([
38
+ 'create index `index_alt_on_a_and_b` on `lhmn_alt` (`a`(10), `b`)'
39
+ ])
40
+ end
41
+
42
+ it 'should add an index with a custom name' do
43
+ @creator.add_index([:a, :b], :custom_index_name)
44
+
45
+ @creator.statements.must_equal([
46
+ 'create index `custom_index_name` on `lhmn_alt` (`a`, `b`)'
47
+ ])
48
+ end
49
+
50
+ it 'should raise an error when the index name is not a string or symbol' do
51
+ assert_raises ArgumentError do
52
+ @creator.add_index([:a, :b], :name => :custom_index_name)
53
+ end
54
+ end
55
+
56
+ it 'should add a unique index' do
57
+ @creator.add_unique_index(['a(5)', :b])
58
+
59
+ @creator.statements.must_equal([
60
+ 'create unique index `index_alt_on_a_and_b` on `lhmn_alt` (`a`(5), `b`)'
61
+ ])
62
+ end
63
+
64
+ it 'should add a unique index with a custom name' do
65
+ @creator.add_unique_index([:a, :b], :custom_index_name)
66
+
67
+ @creator.statements.must_equal([
68
+ 'create unique index `custom_index_name` on `lhmn_alt` (`a`, `b`)'
69
+ ])
70
+ end
71
+
72
+ it 'should raise an error when the unique index name is not a string or symbol' do
73
+ assert_raises ArgumentError do
74
+ @creator.add_unique_index([:a, :b], :name => :custom_index_name)
75
+ end
76
+ end
77
+
78
+ it 'should remove an index' do
79
+ @creator.remove_index(['b', 'a'])
80
+
81
+ @creator.statements.must_equal([
82
+ 'drop index `index_alt_on_b_and_a` on `lhmn_alt`'
83
+ ])
84
+ end
85
+
86
+ it 'should remove an index with a custom name' do
87
+ @creator.remove_index([:a, :b], :custom_index_name)
88
+
89
+ @creator.statements.must_equal([
90
+ 'drop index `custom_index_name` on `lhmn_alt`'
91
+ ])
92
+ end
93
+ end
94
+
95
+ describe 'column changes' do
96
+ it 'should add a column' do
97
+ @creator.add_column('logins', 'INT(12)')
98
+
99
+ @creator.statements.must_equal([
100
+ 'alter table `lhmn_alt` add column `logins` INT(12)'
101
+ ])
102
+ end
103
+
104
+ it 'should remove a column' do
105
+ @creator.remove_column('logins')
106
+
107
+ @creator.statements.must_equal([
108
+ 'alter table `lhmn_alt` drop `logins`'
109
+ ])
110
+ end
111
+
112
+ it 'should change a column' do
113
+ @creator.change_column('logins', 'INT(11)')
114
+
115
+ @creator.statements.must_equal([
116
+ 'alter table `lhmn_alt` modify column `logins` INT(11)'
117
+ ])
118
+ end
119
+ end
120
+
121
+ describe 'direct changes' do
122
+ it 'should accept a ddl statement' do
123
+ @creator.ddl('alter table `%s` add column `f` tinyint(1)' % @creator.name)
124
+
125
+ @creator.statements.must_equal([
126
+ 'alter table `lhmn_alt` add column `f` tinyint(1)'
127
+ ])
128
+ end
129
+ end
130
+
131
+ describe 'multiple changes' do
132
+ it 'should add two columns' do
133
+ @creator.add_column('first', 'VARCHAR(64)')
134
+ @creator.add_column('last', 'VARCHAR(64)')
135
+ @creator.statements.length.must_equal(2)
136
+
137
+ @creator.
138
+ statements[0].
139
+ must_equal('alter table `lhmn_alt` add column `first` VARCHAR(64)')
140
+
141
+ @creator.
142
+ statements[1].
143
+ must_equal('alter table `lhmn_alt` add column `last` VARCHAR(64)')
144
+ end
145
+ end
146
+ end
@@ -0,0 +1,97 @@
1
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
2
+
3
+ require 'lhm/printer'
4
+
5
+ describe Lhm::Printer do
6
+ include UnitHelper
7
+
8
+ describe 'percentage printer' do
9
+
10
+ before(:each) do
11
+ @printer = Lhm::Printer::Percentage.new
12
+ end
13
+
14
+ it 'prints the percentage' do
15
+ mock = MiniTest::Mock.new
16
+ 10.times do |i|
17
+ mock.expect(:write, :return_value) do |message|
18
+ message = message.first if message.is_a?(Array)
19
+ assert_match(/^\r/, message)
20
+ assert_match(/#{i}\/10/, message)
21
+ end
22
+ end
23
+
24
+ @printer.instance_variable_set(:@output, mock)
25
+ 10.times { |i| @printer.notify(i, 10) }
26
+ mock.verify
27
+ end
28
+
29
+ it 'always prints a bigger message' do
30
+ @length = 0
31
+ printer_mock = mock()
32
+ printer_mock.expects(:write).at_least_once
33
+
34
+ def assert_length(printer)
35
+ new_length = printer.instance_variable_get(:@max_length)
36
+ assert new_length >= @length
37
+ @length = new_length
38
+ end
39
+
40
+ @printer.instance_variable_set(:@output, printer_mock)
41
+ @printer.notify(10, 100)
42
+ assert_length(@printer)
43
+ @printer.notify(0, 100)
44
+ assert_length(@printer)
45
+ @printer.notify(1, 1000000)
46
+ assert_length(@printer)
47
+ @printer.notify(0, 0)
48
+ assert_length(@printer)
49
+ @printer.notify(0, nil)
50
+ assert_length(@printer)
51
+ end
52
+
53
+ it 'prints the end message' do
54
+ mock = MiniTest::Mock.new
55
+ mock.expect(:write, :return_value, [String])
56
+ mock.expect(:write, :return_value, ["\n"])
57
+
58
+ @printer.instance_variable_set(:@output, mock)
59
+ @printer.end
60
+
61
+ mock.verify
62
+ end
63
+
64
+ it 'prints the exception message' do
65
+ mock = MiniTest::Mock.new
66
+ mock.expect(:write, :return_value, ["\rfailed: woops"])
67
+ mock.expect(:write, :return_value, ["\n"])
68
+
69
+ e = StandardError.new('woops')
70
+
71
+ @printer.instance_variable_set(:@output, mock)
72
+ @printer.exception(e)
73
+
74
+ mock.verify
75
+ end
76
+ end
77
+
78
+ describe 'dot printer' do
79
+
80
+ before(:each) do
81
+ @printer = Lhm::Printer::Dot.new
82
+ end
83
+
84
+ it 'prints the dots' do
85
+ mock = MiniTest::Mock.new
86
+ 10.times do
87
+ mock.expect(:write, :return_value, ['.'])
88
+ end
89
+
90
+ @printer.instance_variable_set(:@output, mock)
91
+ 10.times { @printer.notify }
92
+
93
+ mock.verify
94
+ end
95
+
96
+ end
97
+ end
@@ -0,0 +1,32 @@
1
+ # Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
5
+
6
+ require 'lhm/sql_helper'
7
+
8
+ describe Lhm::SqlHelper do
9
+ it 'should name index with a single column' do
10
+ Lhm::SqlHelper.
11
+ idx_name(:users, :name).
12
+ must_equal('index_users_on_name')
13
+ end
14
+
15
+ it 'should name index with multiple columns' do
16
+ Lhm::SqlHelper.
17
+ idx_name(:users, [:name, :firstname]).
18
+ must_equal('index_users_on_name_and_firstname')
19
+ end
20
+
21
+ it 'should name index with prefixed column' do
22
+ Lhm::SqlHelper.
23
+ idx_name(:tracks, ['title(10)', 'album']).
24
+ must_equal('index_tracks_on_title_and_album')
25
+ end
26
+
27
+ it 'should quote column names in index specification' do
28
+ Lhm::SqlHelper.
29
+ idx_spec(['title(10)', 'album']).
30
+ must_equal('`title`(10), `album`')
31
+ end
32
+ end
@@ -0,0 +1,39 @@
1
+ require 'test_helper'
2
+
3
+ describe Lhm::TableName do
4
+ describe "#archived" do
5
+ it "prefixes and timestamps the old table" do
6
+ subject = Lhm::TableName.new("original", Time.new(2000,01,02,03,04,05))
7
+ assert_equal "lhma_2000_01_02_03_04_05_000_original", subject.archived
8
+ end
9
+
10
+ it "truncates names below 64 characters" do
11
+ subject = Lhm::TableName.new("some_very_long_original_table_name_that_exceeds_64_characters", Time.new(2000,01,02,03,04,05))
12
+ assert_equal "lhma_2000_01_02_03_04_05_000_some_very_long_original_table_name_", subject.archived
13
+ end
14
+ end
15
+
16
+ describe "#failed" do
17
+ it "prefixes and postfixes and timestamps the old table" do
18
+ subject = Lhm::TableName.new("original", Time.new(2000,01,02,03,04,05))
19
+ assert_equal "lhma_2000_01_02_03_04_05_000_original_failed", subject.failed
20
+ end
21
+
22
+ it "truncates names below 64 characters" do
23
+ subject = Lhm::TableName.new("some_very_long_original_table_name_that_exceeds_64_characters", Time.new(2000,01,02,03,04,05))
24
+ assert_equal "lhma_2000_01_02_03_04_05_000_some_very_long_original_tabl_failed", subject.failed
25
+ end
26
+ end
27
+
28
+ describe "#new" do
29
+ it "prefixes and postfixes and timestamps the old table" do
30
+ subject = Lhm::TableName.new("original")
31
+ assert_equal "lhmn_original", subject.new
32
+ end
33
+
34
+ it "truncates names below 64 characters" do
35
+ subject = Lhm::TableName.new("some_very_long_original_table_name_that_exceeds_64_characters")
36
+ assert_equal "lhmn_some_very_long_original_table_name_that_exceeds_64_characte", subject.new
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,47 @@
1
+ # Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
5
+
6
+ require 'lhm/table'
7
+
8
+ describe Lhm::Table do
9
+ include UnitHelper
10
+
11
+ describe 'names' do
12
+ it 'should name destination' do
13
+ @table = Lhm::Table.new('users')
14
+ @table.destination_name.must_equal 'lhmn_users'
15
+ end
16
+ end
17
+
18
+ describe 'constraints' do
19
+ def set_columns(table, columns)
20
+ table.instance_variable_set('@columns', columns)
21
+ end
22
+
23
+ it 'should be satisfied with a single column primary key called id' do
24
+ @table = Lhm::Table.new('table', 'id')
25
+ set_columns(@table, { 'id' => { :type => 'int(1)' } })
26
+ @table.satisfies_id_column_requirement?.must_equal true
27
+ end
28
+
29
+ it 'should be satisfied with a primary key not called id, as long as there is still an id' do
30
+ @table = Lhm::Table.new('table', 'uuid')
31
+ set_columns(@table, { 'id' => { :type => 'int(1)' } })
32
+ @table.satisfies_id_column_requirement?.must_equal true
33
+ end
34
+
35
+ it 'should be satisifed if display attributes are not present (deprecated in mysql 8)' do
36
+ @table = Lhm::Table.new('table', 'id')
37
+ set_columns(@table, { 'id' => { :type => 'int' } })
38
+ @table.satisfies_id_column_requirement?.must_equal true
39
+ end
40
+
41
+ it 'should not be satisfied if id is not numeric' do
42
+ @table = Lhm::Table.new('table', 'id')
43
+ set_columns(@table, { 'id' => { :type => 'varchar(255)' } })
44
+ @table.satisfies_id_column_requirement?.must_equal false
45
+ end
46
+ end
47
+ end