lhm-shopify 3.3.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. checksums.yaml +7 -0
  2. data/.github/workflows/test.yml +34 -0
  3. data/.gitignore +17 -0
  4. data/.rubocop.yml +183 -0
  5. data/.travis.yml +21 -0
  6. data/CHANGELOG.md +216 -0
  7. data/Gemfile +5 -0
  8. data/LICENSE +27 -0
  9. data/README.md +284 -0
  10. data/Rakefile +22 -0
  11. data/bin/.gitkeep +0 -0
  12. data/dbdeployer/config.json +32 -0
  13. data/dbdeployer/install.sh +64 -0
  14. data/dev.yml +20 -0
  15. data/gemfiles/ar-2.3_mysql.gemfile +6 -0
  16. data/gemfiles/ar-3.2_mysql.gemfile +5 -0
  17. data/gemfiles/ar-3.2_mysql2.gemfile +5 -0
  18. data/gemfiles/ar-4.0_mysql2.gemfile +5 -0
  19. data/gemfiles/ar-4.1_mysql2.gemfile +5 -0
  20. data/gemfiles/ar-4.2_mysql2.gemfile +5 -0
  21. data/gemfiles/ar-5.0_mysql2.gemfile +5 -0
  22. data/lhm.gemspec +34 -0
  23. data/lib/lhm.rb +131 -0
  24. data/lib/lhm/atomic_switcher.rb +52 -0
  25. data/lib/lhm/chunk_finder.rb +32 -0
  26. data/lib/lhm/chunk_insert.rb +51 -0
  27. data/lib/lhm/chunker.rb +87 -0
  28. data/lib/lhm/cleanup/current.rb +74 -0
  29. data/lib/lhm/command.rb +48 -0
  30. data/lib/lhm/entangler.rb +117 -0
  31. data/lib/lhm/intersection.rb +51 -0
  32. data/lib/lhm/invoker.rb +98 -0
  33. data/lib/lhm/locked_switcher.rb +74 -0
  34. data/lib/lhm/migration.rb +43 -0
  35. data/lib/lhm/migrator.rb +237 -0
  36. data/lib/lhm/printer.rb +59 -0
  37. data/lib/lhm/railtie.rb +9 -0
  38. data/lib/lhm/sql_helper.rb +77 -0
  39. data/lib/lhm/sql_retry.rb +61 -0
  40. data/lib/lhm/table.rb +121 -0
  41. data/lib/lhm/table_name.rb +23 -0
  42. data/lib/lhm/test_support.rb +35 -0
  43. data/lib/lhm/throttler.rb +36 -0
  44. data/lib/lhm/throttler/slave_lag.rb +145 -0
  45. data/lib/lhm/throttler/threads_running.rb +53 -0
  46. data/lib/lhm/throttler/time.rb +29 -0
  47. data/lib/lhm/timestamp.rb +11 -0
  48. data/lib/lhm/version.rb +6 -0
  49. data/shipit.rubygems.yml +0 -0
  50. data/spec/.lhm.example +4 -0
  51. data/spec/README.md +58 -0
  52. data/spec/fixtures/bigint_table.ddl +4 -0
  53. data/spec/fixtures/composite_primary_key.ddl +7 -0
  54. data/spec/fixtures/custom_primary_key.ddl +6 -0
  55. data/spec/fixtures/destination.ddl +6 -0
  56. data/spec/fixtures/lines.ddl +7 -0
  57. data/spec/fixtures/origin.ddl +6 -0
  58. data/spec/fixtures/permissions.ddl +5 -0
  59. data/spec/fixtures/small_table.ddl +4 -0
  60. data/spec/fixtures/tracks.ddl +5 -0
  61. data/spec/fixtures/users.ddl +14 -0
  62. data/spec/fixtures/wo_id_int_column.ddl +6 -0
  63. data/spec/integration/atomic_switcher_spec.rb +93 -0
  64. data/spec/integration/chunk_insert_spec.rb +29 -0
  65. data/spec/integration/chunker_spec.rb +185 -0
  66. data/spec/integration/cleanup_spec.rb +136 -0
  67. data/spec/integration/entangler_spec.rb +66 -0
  68. data/spec/integration/integration_helper.rb +237 -0
  69. data/spec/integration/invoker_spec.rb +33 -0
  70. data/spec/integration/lhm_spec.rb +585 -0
  71. data/spec/integration/lock_wait_timeout_spec.rb +30 -0
  72. data/spec/integration/locked_switcher_spec.rb +50 -0
  73. data/spec/integration/sql_retry/lock_wait_spec.rb +125 -0
  74. data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +101 -0
  75. data/spec/integration/table_spec.rb +91 -0
  76. data/spec/test_helper.rb +32 -0
  77. data/spec/unit/atomic_switcher_spec.rb +31 -0
  78. data/spec/unit/chunk_finder_spec.rb +73 -0
  79. data/spec/unit/chunk_insert_spec.rb +44 -0
  80. data/spec/unit/chunker_spec.rb +166 -0
  81. data/spec/unit/entangler_spec.rb +124 -0
  82. data/spec/unit/intersection_spec.rb +51 -0
  83. data/spec/unit/lhm_spec.rb +29 -0
  84. data/spec/unit/locked_switcher_spec.rb +51 -0
  85. data/spec/unit/migrator_spec.rb +146 -0
  86. data/spec/unit/printer_spec.rb +97 -0
  87. data/spec/unit/sql_helper_spec.rb +32 -0
  88. data/spec/unit/table_name_spec.rb +39 -0
  89. data/spec/unit/table_spec.rb +47 -0
  90. data/spec/unit/throttler/slave_lag_spec.rb +317 -0
  91. data/spec/unit/throttler/threads_running_spec.rb +64 -0
  92. data/spec/unit/throttler_spec.rb +124 -0
  93. data/spec/unit/unit_helper.rb +13 -0
  94. metadata +239 -0
@@ -0,0 +1,29 @@
1
+ # Copyright (c) 2011 - 2013, SoundCloud Ltd.
2
+
3
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
4
+
5
+ describe Lhm do
6
+
7
+ before(:each) do
8
+ Lhm.remove_class_variable :@@logger if Lhm.class_variable_defined? :@@logger
9
+ end
10
+
11
+ describe 'logger' do
12
+
13
+ it 'should use the default parameters if no logger explicitly set' do
14
+ Lhm.logger.must_be_kind_of Logger
15
+ Lhm.logger.level.must_equal Logger::INFO
16
+ Lhm.logger.instance_eval { @logdev }.dev.must_equal STDOUT
17
+ end
18
+
19
+ it 'should use s new logger if set' do
20
+ l = Logger.new('omg.ponies')
21
+ l.level = Logger::ERROR
22
+ Lhm.logger = l
23
+
24
+ Lhm.logger.level.must_equal Logger::ERROR
25
+ Lhm.logger.instance_eval { @logdev }.dev.must_be_kind_of File
26
+ Lhm.logger.instance_eval { @logdev }.dev.path.must_equal 'omg.ponies'
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,51 @@
1
+ # Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
5
+
6
+ require 'lhm/table'
7
+ require 'lhm/migration'
8
+ require 'lhm/locked_switcher'
9
+
10
+ describe Lhm::LockedSwitcher do
11
+ include UnitHelper
12
+
13
+ before(:each) do
14
+ @start = Time.now
15
+ @origin = Lhm::Table.new('origin')
16
+ @destination = Lhm::Table.new('destination')
17
+ @migration = Lhm::Migration.new(@origin, @destination, @start)
18
+ @switcher = Lhm::LockedSwitcher.new(@migration, nil)
19
+ end
20
+
21
+ describe 'uncommitted' do
22
+ it 'should disable autocommit first' do
23
+ @switcher.
24
+ statements[0..1].
25
+ must_equal([
26
+ 'set @lhm_auto_commit = @@session.autocommit',
27
+ 'set session autocommit = 0'
28
+ ])
29
+ end
30
+
31
+ it 'should reapply original autocommit settings at the end' do
32
+ @switcher.
33
+ statements[-1].
34
+ must_equal('set session autocommit = @lhm_auto_commit')
35
+ end
36
+ end
37
+
38
+ describe 'switch' do
39
+ it 'should lock origin and destination table, switch, commit and unlock' do
40
+ @switcher.
41
+ switch.
42
+ must_equal([
43
+ 'lock table `origin` write, `destination` write',
44
+ "alter table `origin` rename `#{ @migration.archive_name }`",
45
+ 'alter table `destination` rename `origin`',
46
+ 'commit',
47
+ 'unlock tables'
48
+ ])
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,146 @@
1
+ # Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
5
+
6
+ require 'lhm/table'
7
+ require 'lhm/migrator'
8
+
9
+ describe Lhm::Migrator do
10
+ include UnitHelper
11
+
12
+ before(:each) do
13
+ @table = Lhm::Table.new('alt')
14
+ @creator = Lhm::Migrator.new(@table)
15
+ end
16
+
17
+ describe 'index changes' do
18
+ it 'should add an index' do
19
+ @creator.add_index(:a)
20
+
21
+ @creator.statements.must_equal([
22
+ 'create index `index_alt_on_a` on `lhmn_alt` (`a`)'
23
+ ])
24
+ end
25
+
26
+ it 'should add a composite index' do
27
+ @creator.add_index([:a, :b])
28
+
29
+ @creator.statements.must_equal([
30
+ 'create index `index_alt_on_a_and_b` on `lhmn_alt` (`a`, `b`)'
31
+ ])
32
+ end
33
+
34
+ it 'should add an index with prefix length' do
35
+ @creator.add_index(['a(10)', 'b'])
36
+
37
+ @creator.statements.must_equal([
38
+ 'create index `index_alt_on_a_and_b` on `lhmn_alt` (`a`(10), `b`)'
39
+ ])
40
+ end
41
+
42
+ it 'should add an index with a custom name' do
43
+ @creator.add_index([:a, :b], :custom_index_name)
44
+
45
+ @creator.statements.must_equal([
46
+ 'create index `custom_index_name` on `lhmn_alt` (`a`, `b`)'
47
+ ])
48
+ end
49
+
50
+ it 'should raise an error when the index name is not a string or symbol' do
51
+ assert_raises ArgumentError do
52
+ @creator.add_index([:a, :b], :name => :custom_index_name)
53
+ end
54
+ end
55
+
56
+ it 'should add a unique index' do
57
+ @creator.add_unique_index(['a(5)', :b])
58
+
59
+ @creator.statements.must_equal([
60
+ 'create unique index `index_alt_on_a_and_b` on `lhmn_alt` (`a`(5), `b`)'
61
+ ])
62
+ end
63
+
64
+ it 'should add a unique index with a custom name' do
65
+ @creator.add_unique_index([:a, :b], :custom_index_name)
66
+
67
+ @creator.statements.must_equal([
68
+ 'create unique index `custom_index_name` on `lhmn_alt` (`a`, `b`)'
69
+ ])
70
+ end
71
+
72
+ it 'should raise an error when the unique index name is not a string or symbol' do
73
+ assert_raises ArgumentError do
74
+ @creator.add_unique_index([:a, :b], :name => :custom_index_name)
75
+ end
76
+ end
77
+
78
+ it 'should remove an index' do
79
+ @creator.remove_index(['b', 'a'])
80
+
81
+ @creator.statements.must_equal([
82
+ 'drop index `index_alt_on_b_and_a` on `lhmn_alt`'
83
+ ])
84
+ end
85
+
86
+ it 'should remove an index with a custom name' do
87
+ @creator.remove_index([:a, :b], :custom_index_name)
88
+
89
+ @creator.statements.must_equal([
90
+ 'drop index `custom_index_name` on `lhmn_alt`'
91
+ ])
92
+ end
93
+ end
94
+
95
+ describe 'column changes' do
96
+ it 'should add a column' do
97
+ @creator.add_column('logins', 'INT(12)')
98
+
99
+ @creator.statements.must_equal([
100
+ 'alter table `lhmn_alt` add column `logins` INT(12)'
101
+ ])
102
+ end
103
+
104
+ it 'should remove a column' do
105
+ @creator.remove_column('logins')
106
+
107
+ @creator.statements.must_equal([
108
+ 'alter table `lhmn_alt` drop `logins`'
109
+ ])
110
+ end
111
+
112
+ it 'should change a column' do
113
+ @creator.change_column('logins', 'INT(11)')
114
+
115
+ @creator.statements.must_equal([
116
+ 'alter table `lhmn_alt` modify column `logins` INT(11)'
117
+ ])
118
+ end
119
+ end
120
+
121
+ describe 'direct changes' do
122
+ it 'should accept a ddl statement' do
123
+ @creator.ddl('alter table `%s` add column `f` tinyint(1)' % @creator.name)
124
+
125
+ @creator.statements.must_equal([
126
+ 'alter table `lhmn_alt` add column `f` tinyint(1)'
127
+ ])
128
+ end
129
+ end
130
+
131
+ describe 'multiple changes' do
132
+ it 'should add two columns' do
133
+ @creator.add_column('first', 'VARCHAR(64)')
134
+ @creator.add_column('last', 'VARCHAR(64)')
135
+ @creator.statements.length.must_equal(2)
136
+
137
+ @creator.
138
+ statements[0].
139
+ must_equal('alter table `lhmn_alt` add column `first` VARCHAR(64)')
140
+
141
+ @creator.
142
+ statements[1].
143
+ must_equal('alter table `lhmn_alt` add column `last` VARCHAR(64)')
144
+ end
145
+ end
146
+ end
@@ -0,0 +1,97 @@
1
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
2
+
3
+ require 'lhm/printer'
4
+
5
+ describe Lhm::Printer do
6
+ include UnitHelper
7
+
8
+ describe 'percentage printer' do
9
+
10
+ before(:each) do
11
+ @printer = Lhm::Printer::Percentage.new
12
+ end
13
+
14
+ it 'prints the percentage' do
15
+ mock = MiniTest::Mock.new
16
+ 10.times do |i|
17
+ mock.expect(:write, :return_value) do |message|
18
+ message = message.first if message.is_a?(Array)
19
+ assert_match(/^\r/, message)
20
+ assert_match(/#{i}\/10/, message)
21
+ end
22
+ end
23
+
24
+ @printer.instance_variable_set(:@output, mock)
25
+ 10.times { |i| @printer.notify(i, 10) }
26
+ mock.verify
27
+ end
28
+
29
+ it 'always prints a bigger message' do
30
+ @length = 0
31
+ printer_mock = mock()
32
+ printer_mock.expects(:write).at_least_once
33
+
34
+ def assert_length(printer)
35
+ new_length = printer.instance_variable_get(:@max_length)
36
+ assert new_length >= @length
37
+ @length = new_length
38
+ end
39
+
40
+ @printer.instance_variable_set(:@output, printer_mock)
41
+ @printer.notify(10, 100)
42
+ assert_length(@printer)
43
+ @printer.notify(0, 100)
44
+ assert_length(@printer)
45
+ @printer.notify(1, 1000000)
46
+ assert_length(@printer)
47
+ @printer.notify(0, 0)
48
+ assert_length(@printer)
49
+ @printer.notify(0, nil)
50
+ assert_length(@printer)
51
+ end
52
+
53
+ it 'prints the end message' do
54
+ mock = MiniTest::Mock.new
55
+ mock.expect(:write, :return_value, [String])
56
+ mock.expect(:write, :return_value, ["\n"])
57
+
58
+ @printer.instance_variable_set(:@output, mock)
59
+ @printer.end
60
+
61
+ mock.verify
62
+ end
63
+
64
+ it 'prints the exception message' do
65
+ mock = MiniTest::Mock.new
66
+ mock.expect(:write, :return_value, ["\rfailed: woops"])
67
+ mock.expect(:write, :return_value, ["\n"])
68
+
69
+ e = StandardError.new('woops')
70
+
71
+ @printer.instance_variable_set(:@output, mock)
72
+ @printer.exception(e)
73
+
74
+ mock.verify
75
+ end
76
+ end
77
+
78
+ describe 'dot printer' do
79
+
80
+ before(:each) do
81
+ @printer = Lhm::Printer::Dot.new
82
+ end
83
+
84
+ it 'prints the dots' do
85
+ mock = MiniTest::Mock.new
86
+ 10.times do
87
+ mock.expect(:write, :return_value, ['.'])
88
+ end
89
+
90
+ @printer.instance_variable_set(:@output, mock)
91
+ 10.times { @printer.notify }
92
+
93
+ mock.verify
94
+ end
95
+
96
+ end
97
+ end
@@ -0,0 +1,32 @@
1
+ # Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
5
+
6
+ require 'lhm/sql_helper'
7
+
8
+ describe Lhm::SqlHelper do
9
+ it 'should name index with a single column' do
10
+ Lhm::SqlHelper.
11
+ idx_name(:users, :name).
12
+ must_equal('index_users_on_name')
13
+ end
14
+
15
+ it 'should name index with multiple columns' do
16
+ Lhm::SqlHelper.
17
+ idx_name(:users, [:name, :firstname]).
18
+ must_equal('index_users_on_name_and_firstname')
19
+ end
20
+
21
+ it 'should name index with prefixed column' do
22
+ Lhm::SqlHelper.
23
+ idx_name(:tracks, ['title(10)', 'album']).
24
+ must_equal('index_tracks_on_title_and_album')
25
+ end
26
+
27
+ it 'should quote column names in index specification' do
28
+ Lhm::SqlHelper.
29
+ idx_spec(['title(10)', 'album']).
30
+ must_equal('`title`(10), `album`')
31
+ end
32
+ end
@@ -0,0 +1,39 @@
1
+ require 'test_helper'
2
+
3
+ describe Lhm::TableName do
4
+ describe "#archived" do
5
+ it "prefixes and timestamps the old table" do
6
+ subject = Lhm::TableName.new("original", Time.new(2000,01,02,03,04,05))
7
+ assert_equal "lhma_2000_01_02_03_04_05_000_original", subject.archived
8
+ end
9
+
10
+ it "truncates names below 64 characters" do
11
+ subject = Lhm::TableName.new("some_very_long_original_table_name_that_exceeds_64_characters", Time.new(2000,01,02,03,04,05))
12
+ assert_equal "lhma_2000_01_02_03_04_05_000_some_very_long_original_table_name_", subject.archived
13
+ end
14
+ end
15
+
16
+ describe "#failed" do
17
+ it "prefixes and postfixes and timestamps the old table" do
18
+ subject = Lhm::TableName.new("original", Time.new(2000,01,02,03,04,05))
19
+ assert_equal "lhma_2000_01_02_03_04_05_000_original_failed", subject.failed
20
+ end
21
+
22
+ it "truncates names below 64 characters" do
23
+ subject = Lhm::TableName.new("some_very_long_original_table_name_that_exceeds_64_characters", Time.new(2000,01,02,03,04,05))
24
+ assert_equal "lhma_2000_01_02_03_04_05_000_some_very_long_original_tabl_failed", subject.failed
25
+ end
26
+ end
27
+
28
+ describe "#new" do
29
+ it "prefixes and postfixes and timestamps the old table" do
30
+ subject = Lhm::TableName.new("original")
31
+ assert_equal "lhmn_original", subject.new
32
+ end
33
+
34
+ it "truncates names below 64 characters" do
35
+ subject = Lhm::TableName.new("some_very_long_original_table_name_that_exceeds_64_characters")
36
+ assert_equal "lhmn_some_very_long_original_table_name_that_exceeds_64_characte", subject.new
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,47 @@
1
+ # Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
5
+
6
+ require 'lhm/table'
7
+
8
+ describe Lhm::Table do
9
+ include UnitHelper
10
+
11
+ describe 'names' do
12
+ it 'should name destination' do
13
+ @table = Lhm::Table.new('users')
14
+ @table.destination_name.must_equal 'lhmn_users'
15
+ end
16
+ end
17
+
18
+ describe 'constraints' do
19
+ def set_columns(table, columns)
20
+ table.instance_variable_set('@columns', columns)
21
+ end
22
+
23
+ it 'should be satisfied with a single column primary key called id' do
24
+ @table = Lhm::Table.new('table', 'id')
25
+ set_columns(@table, { 'id' => { :type => 'int(1)' } })
26
+ @table.satisfies_id_column_requirement?.must_equal true
27
+ end
28
+
29
+ it 'should be satisfied with a primary key not called id, as long as there is still an id' do
30
+ @table = Lhm::Table.new('table', 'uuid')
31
+ set_columns(@table, { 'id' => { :type => 'int(1)' } })
32
+ @table.satisfies_id_column_requirement?.must_equal true
33
+ end
34
+
35
+ it 'should be satisifed if display attributes are not present (deprecated in mysql 8)' do
36
+ @table = Lhm::Table.new('table', 'id')
37
+ set_columns(@table, { 'id' => { :type => 'int' } })
38
+ @table.satisfies_id_column_requirement?.must_equal true
39
+ end
40
+
41
+ it 'should not be satisfied if id is not numeric' do
42
+ @table = Lhm::Table.new('table', 'id')
43
+ set_columns(@table, { 'id' => { :type => 'varchar(255)' } })
44
+ @table.satisfies_id_column_requirement?.must_equal false
45
+ end
46
+ end
47
+ end