synchronised_migration 2.1.1 → 3.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -0
- data/README.md +47 -27
- data/exe/synchronised-migration +7 -0
- data/lib/synchronised_migration/commands.rb +109 -0
- data/lib/synchronised_migration/configuration.rb +50 -0
- data/lib/synchronised_migration/main.rb +63 -123
- data/lib/synchronised_migration/result.rb +45 -14
- data/lib/synchronised_migration/version.rb +1 -1
- data/lib/synchronised_migration.rb +11 -19
- metadata +84 -23
- data/.gitignore +0 -5
- data/.ruby-version +0 -1
- data/.travis.yml +0 -10
- data/Gemfile +0 -2
- data/LICENSE +0 -21
- data/Rakefile +0 -2
- data/lib/synchronised_migration/railtie.rb +0 -5
- data/lib/tasks/synchronised_migration.rake +0 -9
- data/spec/spec_helper.rb +0 -9
- data/spec/synchronised_migration/main_spec.rb +0 -164
- data/synchronised_migration.gemspec +0 -24
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 460b790e8a1c958ba08357abfaa6927c5da2b969a5d7a83c3a31c33125dfbeaa
|
4
|
+
data.tar.gz: d4eb86d31a9803a0077a952aa04b26e6b3bc063e52c0efcc871dce6ec72a1d53
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4b8a0b703645774490fea5ae09115e43772543dbe2d629c73eea4fec2a1dc8a2e48ba12a10ea9387d4894f9fa893f9f2d402fd22e5a82ed000564a7d82ea51c8
|
7
|
+
data.tar.gz: b9449bf96097a76f23d617cfaca864389caad35e94402d89b459333f839f250ce43d9780e3327d2526a10be74f9c5304ecb9ccbff74230886aedced7c8bad4eb
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,22 @@
|
|
1
1
|
# Synchronised Migration
|
2
2
|
|
3
|
+
|
4
|
+
## 3.1.0
|
5
|
+
|
6
|
+
- [PLAT-183] Ruby 3.1 and publish coverage with github action
|
7
|
+
|
8
|
+
## 3.0.0
|
9
|
+
|
10
|
+
- [OPS-201] Migrate library to CLI tool
|
11
|
+
|
12
|
+
## 2.2.0
|
13
|
+
|
14
|
+
- [TT-8617] Update to build with github actions / ruby 3.0
|
15
|
+
|
16
|
+
## 2.1.2
|
17
|
+
|
18
|
+
* [TT-7511] Use the correct `ex` option for key value expiry times instead of `ttl`
|
19
|
+
|
3
20
|
## 2.1.1
|
4
21
|
|
5
22
|
* [TT-5896] Early exit from migration inside lock if already completed
|
data/README.md
CHANGED
@@ -1,50 +1,70 @@
|
|
1
1
|
# Synchronised Migration
|
2
2
|
|
3
|
-
[![
|
3
|
+
[![Gem Version](https://badge.fury.io/rb/synchronised_migration.svg)](http://badge.fury.io/rb/synchronised_migration)
|
4
|
+
[![Build Status](https://github.com/sealink/synchronised-migration-rb/workflows/Build%20and%20Test/badge.svg?branch=master)](https://github.com/sealink/synchronised-migration-rb/actions)
|
4
5
|
|
5
6
|
This gem makes it possible to deploy multiple instances with data migration
|
6
|
-
simultaneously.
|
7
|
+
simultaneously. It uses Redis to ensure that there will be only one migration
|
7
8
|
running.
|
8
9
|
|
9
|
-
|
10
|
-
Ruby projects so long as you load the rake task in Rakefile instead of relying
|
11
|
-
on Railtie.
|
10
|
+
## Usage
|
12
11
|
|
13
|
-
|
14
|
-
Docker](https://github.com/sealink/craft-docker) project.
|
12
|
+
### Migrating an application
|
15
13
|
|
16
|
-
|
14
|
+
```
|
15
|
+
bundle exec synchronised-migration migrate --config=migration.yml --version=1.2.3
|
16
|
+
```
|
17
17
|
|
18
|
-
|
18
|
+
### Checking lock stats
|
19
19
|
|
20
20
|
```
|
21
|
-
|
22
|
-
config.host = 'example.com'
|
23
|
-
config.port = 6379
|
24
|
-
config.db = 0
|
25
|
-
end
|
21
|
+
bundle exec synchronised-migration status --config=migration.yml --version=1.2.3
|
26
22
|
```
|
27
23
|
|
28
|
-
|
29
|
-
|
24
|
+
### Clearing locks
|
25
|
+
|
26
|
+
If a previous migration has failed a new one will not be allowed to start
|
27
|
+
unless it is first cleared.
|
28
|
+
|
29
|
+
```
|
30
|
+
bundle exec synchronised-migration clear --config=migration.yml
|
31
|
+
```
|
30
32
|
|
31
|
-
|
33
|
+
If you wish to clear all locks, \*including the running lock then you must also provide version
|
32
34
|
|
33
35
|
```
|
34
|
-
|
35
|
-
WITH_CLEAN_BUNDLER_ENV=1 # Non-empty for true
|
36
|
-
REDLOCK_TIMEOUT_MS=3600000
|
37
|
-
REDLOCK_RETRY_DELAY_MS=200
|
38
|
-
REDLOCK_LOCK_KEY=migration-in-progress
|
39
|
-
REDLOCK_FAIL_KEY=migration-failed
|
36
|
+
bundle exec synchronised-migration clear --config=migration.yml --version=1.2.3 --all
|
40
37
|
```
|
41
38
|
|
42
|
-
|
39
|
+
### Configuration
|
40
|
+
|
41
|
+
| Key | Description |
|
42
|
+
| ------------------------ | --------------------------------------- |
|
43
|
+
| redis_uri | Redis server to use for locking |
|
44
|
+
| application | Application being migrated |
|
45
|
+
| version | Version being migrated too |
|
46
|
+
| debug | Enable additional debug information |
|
47
|
+
| command | Command to execute for the migration |
|
48
|
+
| timeout_milliseconds | Timeout to wait for the lock |
|
49
|
+
| retry_delay_milliseconds | Retry for the lock every x milliseconds |
|
50
|
+
|
51
|
+
### Overriding config file options
|
52
|
+
|
53
|
+
You may override certain config options such as the `redis-uri` and the ```debug``` flag
|
54
|
+
by providing them via cli options.
|
43
55
|
|
44
56
|
```
|
45
|
-
|
57
|
+
bundle exec synchronised-migration migrate --config=migration.yml --version=1.2.3 --redis_uri=redis://127.0.0.1:6379/0
|
46
58
|
```
|
47
59
|
|
48
|
-
##
|
60
|
+
## Release
|
61
|
+
|
62
|
+
To publish a new version of this gem the following steps must be taken.
|
49
63
|
|
50
|
-
|
64
|
+
- Update the version in the following files
|
65
|
+
```
|
66
|
+
CHANGELOG.md
|
67
|
+
lib/synchronised_migration/version.rb
|
68
|
+
```
|
69
|
+
- Create a tag using the format v0.1.0
|
70
|
+
- Follow build progress in GitHub actions
|
@@ -0,0 +1,109 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SynchronisedMigration
|
4
|
+
module Commands
|
5
|
+
extend Dry::CLI::Registry
|
6
|
+
|
7
|
+
class Migrate < Dry::CLI::Command
|
8
|
+
desc "Run the provided command in a migration lock"
|
9
|
+
|
10
|
+
option :config, type: :string, required: true, desc: "Configuration file location"
|
11
|
+
option :version, type: :string, required: true, desc: "Version we are migrating to"
|
12
|
+
option :application, type: :string, desc: "Application we are migrating"
|
13
|
+
option :redis_uri, type: :string, desc: "Redis Server URI"
|
14
|
+
option :debug, type: :boolean, desc: "Enable additional debugging output"
|
15
|
+
|
16
|
+
def call(**options)
|
17
|
+
abort "Config location must be provided" if options[:config].nil?
|
18
|
+
abort "Version must be provided" if options[:version].nil?
|
19
|
+
|
20
|
+
config = SynchronisedMigration::Configuration.from_cli(options)
|
21
|
+
result = SynchronisedMigration::Main.new(config).call
|
22
|
+
|
23
|
+
abort(result.error_msg) if result.failure?
|
24
|
+
$stdout.puts "Complete!"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
class Clear < Dry::CLI::Command
|
29
|
+
desc "Clear the migration keys"
|
30
|
+
|
31
|
+
option :config, type: :string, required: true, desc: "Configuration file"
|
32
|
+
option :version, type: :string, required: true, desc: "Version we are migrating to"
|
33
|
+
option :application, type: :string, desc: "Application we are migrating"
|
34
|
+
option :redis_uri, type: :string, desc: "Redis Server URI"
|
35
|
+
option :debug, type: :boolean, desc: "Enable additional debugging output"
|
36
|
+
option :all, type: :boolean, default: false, desc: "Clear all keys including success and lock"
|
37
|
+
|
38
|
+
def call(**options)
|
39
|
+
abort "Config location must be provided" if options[:config].nil?
|
40
|
+
|
41
|
+
config = SynchronisedMigration::Configuration.from_cli(options)
|
42
|
+
|
43
|
+
redis_opts = {url: config.redis_uri}
|
44
|
+
redis_opts[:logger] = Logger.new($stdout) if config.debug?
|
45
|
+
|
46
|
+
redis = Redis.new(redis_opts)
|
47
|
+
redis.del(config.fail_key)
|
48
|
+
|
49
|
+
if options[:all]
|
50
|
+
abort "Version must be provided" if options[:version].nil?
|
51
|
+
redis.del(config.success_key)
|
52
|
+
redis.del(config.lock_key)
|
53
|
+
end
|
54
|
+
|
55
|
+
$stdout.puts "Complete!"
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
class Status < Dry::CLI::Command
|
60
|
+
desc "Shows the current status the redis failure if any"
|
61
|
+
|
62
|
+
option :config, type: :string, required: true, desc: "Configuration file"
|
63
|
+
option :version, type: :string, required: true, desc: "Version we are migrating to"
|
64
|
+
option :application, type: :string, desc: "Application we are migrating"
|
65
|
+
option :redis_uri, type: :string, desc: "Redis Server URI"
|
66
|
+
option :debug, type: :boolean, desc: "Enable additional debugging output"
|
67
|
+
|
68
|
+
def call(**options)
|
69
|
+
abort "Config location must be provided" if options[:config].nil?
|
70
|
+
abort "Version must be provided" if options[:version].nil?
|
71
|
+
|
72
|
+
config = SynchronisedMigration::Configuration.from_cli(options)
|
73
|
+
|
74
|
+
redis_opts = {url: config.redis_uri}
|
75
|
+
redis_opts[:logger] = Logger.new($stdout) if config.debug?
|
76
|
+
redis = Redis.new(redis_opts)
|
77
|
+
|
78
|
+
rows = []
|
79
|
+
fail_key = config.fail_key
|
80
|
+
rows << if redis.exists?(fail_key)
|
81
|
+
[fail_key, "true", "Failed Migration"]
|
82
|
+
else
|
83
|
+
[fail_key, "false", nil]
|
84
|
+
end
|
85
|
+
|
86
|
+
success_key = config.success_key
|
87
|
+
rows << if redis.exists?(success_key)
|
88
|
+
[success_key, "true", "successfully Migrated"]
|
89
|
+
else
|
90
|
+
[success_key, "false", nil]
|
91
|
+
end
|
92
|
+
|
93
|
+
lock_key = config.lock_key
|
94
|
+
rows << if redis.exists?(lock_key)
|
95
|
+
[lock_key, "true", "Running Migration"]
|
96
|
+
else
|
97
|
+
[lock_key, "false", nil]
|
98
|
+
end
|
99
|
+
|
100
|
+
table = TTY::Table.new(["key", "status", "message"], rows)
|
101
|
+
$stdout.puts table.render(:ascii)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
register "migrate", Migrate
|
106
|
+
register "clear", Clear
|
107
|
+
register "status", Status
|
108
|
+
end
|
109
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
module SynchronisedMigration
|
2
|
+
class Configuration
|
3
|
+
attr_accessor :redis_uri
|
4
|
+
attr_accessor :command
|
5
|
+
attr_accessor :application
|
6
|
+
attr_accessor :debug
|
7
|
+
attr_accessor :version
|
8
|
+
|
9
|
+
def self.from_cli(options)
|
10
|
+
config_yml = YAML.load_file(options[:config]).transform_keys(&:to_sym).to_h
|
11
|
+
new(config_yml.merge(options.compact))
|
12
|
+
end
|
13
|
+
|
14
|
+
def initialize(options = {})
|
15
|
+
@application = options.fetch(:application, nil)
|
16
|
+
@version = options.fetch(:version, nil)
|
17
|
+
@redis_uri = options.fetch(:redis_uri, "redis://127.0.0.1:6379/0")
|
18
|
+
@debug = options.fetch(:debug, false)
|
19
|
+
@command = options.fetch(:command, "bin/launch/migrate")
|
20
|
+
@timeout_milliseconds = options.fetch(:timeout_milliseconds, 3_600_000).to_i
|
21
|
+
@retry_delay_milliseconds = options.fetch(:retry_delay_milliseconds, 3000).to_i
|
22
|
+
|
23
|
+
raise OptionParser::InvalidArgument, "Application must be configured" if @application.nil?
|
24
|
+
end
|
25
|
+
|
26
|
+
def timeout_milliseconds
|
27
|
+
@timeout_milliseconds.to_i
|
28
|
+
end
|
29
|
+
|
30
|
+
def retry_delay_milliseconds
|
31
|
+
@retry_delay_milliseconds.to_i
|
32
|
+
end
|
33
|
+
|
34
|
+
def debug?
|
35
|
+
debug
|
36
|
+
end
|
37
|
+
|
38
|
+
def success_key
|
39
|
+
"migration-success-#{application}-#{version}"
|
40
|
+
end
|
41
|
+
|
42
|
+
def fail_key
|
43
|
+
"migration-failed-#{application}"
|
44
|
+
end
|
45
|
+
|
46
|
+
def lock_key
|
47
|
+
"migration-lock-#{application}"
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -1,144 +1,84 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
require 'singleton'
|
1
|
+
module SynchronisedMigration
|
2
|
+
class Main
|
3
|
+
FAIL_TIMEOUT = 3600 # 1 Hour
|
4
|
+
SUCCESS_TIMEOUT = 3600 * 24 * 30 # 30 dyas
|
6
5
|
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
Result = SynchronisedMigration::Result
|
11
|
-
|
12
|
-
class << self
|
13
|
-
extend Forwardable
|
14
|
-
def_delegators :instance, :call
|
15
|
-
end
|
16
|
-
|
17
|
-
def call
|
18
|
-
done_or_execute
|
19
|
-
end
|
20
|
-
|
21
|
-
private
|
22
|
-
|
23
|
-
def done_or_execute
|
24
|
-
return Result.ok if migration_already_completed?
|
25
|
-
lock_and_execute
|
26
|
-
end
|
27
|
-
|
28
|
-
def lock_and_execute
|
29
|
-
redlock.lock! lock_key, timeout do
|
30
|
-
execute
|
6
|
+
def initialize(config)
|
7
|
+
@config = config
|
31
8
|
end
|
32
|
-
end
|
33
9
|
|
34
|
-
|
35
|
-
|
36
|
-
return Result.fail 'Halting the script because the previous migration failed.' if previous_failed?
|
37
|
-
mark_failed
|
38
|
-
migration_success = migrate
|
39
|
-
return Result.fail 'Migration failed.' unless migration_success
|
40
|
-
mark_successful
|
41
|
-
remove_fail_marker
|
42
|
-
return Result.ok
|
43
|
-
end
|
10
|
+
def call
|
11
|
+
return Result.migration_already_completed if migration_already_completed?
|
44
12
|
|
45
|
-
|
46
|
-
return false if !success_key
|
47
|
-
value = redis.get(success_key)
|
48
|
-
not value.nil? and not value.empty?
|
49
|
-
end
|
50
|
-
|
51
|
-
def mark_successful
|
52
|
-
if success_key
|
53
|
-
redis.set success_key, timestamp, ttl: 3600*24*30
|
13
|
+
lock_and_execute
|
54
14
|
end
|
55
|
-
end
|
56
|
-
|
57
|
-
def previous_failed?
|
58
|
-
value = redis.get(fail_key)
|
59
|
-
not value.nil? and not value.empty?
|
60
|
-
end
|
61
|
-
|
62
|
-
def mark_failed
|
63
|
-
redis.set fail_key, timestamp, ttl: 3600
|
64
|
-
end
|
65
15
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
16
|
+
private
|
17
|
+
|
18
|
+
def lock_and_execute
|
19
|
+
client = Redlock::Client.new(
|
20
|
+
[@config.redis_uri], {
|
21
|
+
retry_count: retry_count,
|
22
|
+
retry_delay: @config.retry_delay_milliseconds
|
23
|
+
}
|
24
|
+
)
|
25
|
+
client.lock!(@config.lock_key, @config.timeout_milliseconds) do
|
26
|
+
execute_within_lock
|
74
27
|
end
|
75
|
-
else
|
76
|
-
Kernel.system target_command
|
77
28
|
end
|
78
|
-
$?.success?
|
79
|
-
end
|
80
|
-
|
81
|
-
def with_clean_env?
|
82
|
-
not ENV.fetch('WITH_CLEAN_BUNDLER_ENV', '').empty?
|
83
|
-
end
|
84
29
|
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
30
|
+
def execute_within_lock
|
31
|
+
return Result.ok if migration_already_completed?
|
32
|
+
return Result.previous_migration_failed if previous_failed?
|
33
|
+
mark_failed
|
34
|
+
migration_success = migrate
|
35
|
+
return Result.migration_failed unless migration_success
|
36
|
+
mark_successful
|
37
|
+
remove_fail_marker
|
38
|
+
Result.ok
|
39
|
+
end
|
92
40
|
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
retry_count: retry_count,
|
97
|
-
retry_delay: retry_delay
|
98
|
-
}
|
99
|
-
)
|
100
|
-
end
|
41
|
+
def migration_already_completed?
|
42
|
+
redis.exists?(@config.success_key)
|
43
|
+
end
|
101
44
|
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
SynchronisedMigration.redis_config.port,
|
107
|
-
SynchronisedMigration.redis_config.db
|
108
|
-
)
|
109
|
-
end
|
45
|
+
def mark_successful
|
46
|
+
success_obj = {application: @config.application, version: @config.version, command: @config.command, timestamp: timestamp}.to_json
|
47
|
+
redis.set(@config.success_key, success_obj, ex: SUCCESS_TIMEOUT)
|
48
|
+
end
|
110
49
|
|
111
|
-
|
112
|
-
|
113
|
-
|
50
|
+
def previous_failed?
|
51
|
+
redis.exists?(@config.fail_key)
|
52
|
+
end
|
114
53
|
|
115
|
-
|
116
|
-
|
117
|
-
|
54
|
+
def mark_failed
|
55
|
+
redis.set(@config.fail_key, timestamp, ex: FAIL_TIMEOUT)
|
56
|
+
end
|
118
57
|
|
119
|
-
|
120
|
-
|
121
|
-
|
58
|
+
def remove_fail_marker
|
59
|
+
redis.del(@config.fail_key)
|
60
|
+
end
|
122
61
|
|
123
|
-
|
124
|
-
|
125
|
-
|
62
|
+
def migrate
|
63
|
+
Kernel.system @config.command
|
64
|
+
$?.success?
|
65
|
+
end
|
126
66
|
|
127
|
-
|
128
|
-
|
129
|
-
|
67
|
+
def redis
|
68
|
+
@redis ||= begin
|
69
|
+
redis_opts = {url: @config.redis_uri}
|
70
|
+
redis_opts[:logger] = Logger.new($stdout) if @config.debug?
|
130
71
|
|
131
|
-
|
132
|
-
|
133
|
-
|
72
|
+
Redis.new(redis_opts)
|
73
|
+
end
|
74
|
+
end
|
134
75
|
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
end
|
76
|
+
def timestamp
|
77
|
+
Time.now.to_i
|
78
|
+
end
|
139
79
|
|
140
|
-
|
141
|
-
|
142
|
-
|
80
|
+
def retry_count
|
81
|
+
@config.timeout_milliseconds / @config.retry_delay_milliseconds
|
82
|
+
end
|
143
83
|
end
|
144
84
|
end
|
@@ -1,21 +1,52 @@
|
|
1
|
-
require
|
1
|
+
require "synchronised_migration"
|
2
2
|
|
3
|
-
|
4
|
-
|
3
|
+
module SynchronisedMigration
|
4
|
+
MIGRATION_SUCCESS = 0
|
5
|
+
PREVIOUS_SUCCESS = 1
|
6
|
+
PREVIOUS_FAILED = 2
|
7
|
+
MIGRATION_FAILED = 3
|
5
8
|
|
6
|
-
|
7
|
-
|
8
|
-
end
|
9
|
+
PREVIOUS_FAILED_MSG = "Halting the script because the previous migration failed."
|
10
|
+
MIGRATION_FAILED_MSG = "Migration command failed."
|
9
11
|
|
10
|
-
|
11
|
-
|
12
|
-
end
|
12
|
+
class Result
|
13
|
+
attr_reader :code
|
13
14
|
|
14
|
-
|
15
|
-
|
16
|
-
|
15
|
+
def initialize(code)
|
16
|
+
@code = code
|
17
|
+
end
|
18
|
+
|
19
|
+
def successful?
|
20
|
+
[MIGRATION_SUCCESS, PREVIOUS_SUCCESS].include?(code)
|
21
|
+
end
|
22
|
+
|
23
|
+
def failure?
|
24
|
+
!successful?
|
25
|
+
end
|
26
|
+
|
27
|
+
def error_msg
|
28
|
+
case code
|
29
|
+
when MIGRATION_FAILED
|
30
|
+
MIGRATION_FAILED_MSG
|
31
|
+
when PREVIOUS_FAILED
|
32
|
+
PREVIOUS_FAILED_MSG
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.ok
|
37
|
+
new(MIGRATION_SUCCESS)
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.migration_already_completed
|
41
|
+
new(PREVIOUS_SUCCESS)
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.previous_migration_failed
|
45
|
+
new(PREVIOUS_FAILED)
|
46
|
+
end
|
17
47
|
|
18
|
-
|
19
|
-
|
48
|
+
def self.migration_failed
|
49
|
+
new(MIGRATION_FAILED)
|
50
|
+
end
|
20
51
|
end
|
21
52
|
end
|
@@ -1,22 +1,14 @@
|
|
1
1
|
module SynchronisedMigration
|
2
|
-
class << self
|
3
|
-
attr_accessor :redis_config
|
4
|
-
end
|
5
|
-
|
6
|
-
def self.configure
|
7
|
-
self.redis_config ||= Configuration.new
|
8
|
-
yield(redis_config)
|
9
|
-
end
|
10
|
-
|
11
|
-
class Configuration
|
12
|
-
attr_accessor :host, :port, :db
|
13
|
-
|
14
|
-
def initialize
|
15
|
-
@host = ''
|
16
|
-
@port = 0
|
17
|
-
@db = 0
|
18
|
-
end
|
19
|
-
end
|
20
2
|
end
|
21
3
|
|
22
|
-
require
|
4
|
+
require "redis"
|
5
|
+
require "logger"
|
6
|
+
require "json"
|
7
|
+
require "redlock"
|
8
|
+
require "yaml"
|
9
|
+
require "dry/cli"
|
10
|
+
require "tty/table"
|
11
|
+
require "synchronised_migration/commands"
|
12
|
+
require "synchronised_migration/configuration"
|
13
|
+
require "synchronised_migration/main"
|
14
|
+
require "synchronised_migration/result"
|
metadata
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: synchronised_migration
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 3.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alvin Yim
|
8
8
|
- Stefan Cooper
|
9
9
|
autorequire:
|
10
|
-
bindir:
|
10
|
+
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2022-01-25 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: redlock
|
@@ -25,6 +25,48 @@ dependencies:
|
|
25
25
|
- - ">="
|
26
26
|
- !ruby/object:Gem::Version
|
27
27
|
version: '0.2'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: redis
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - ">="
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: 4.2.1
|
35
|
+
type: :runtime
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - ">="
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: 4.2.1
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: dry-cli
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - ">="
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '0'
|
49
|
+
type: :runtime
|
50
|
+
prerelease: false
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - ">="
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '0'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: tty-table
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '0'
|
63
|
+
type: :runtime
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - ">="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
28
70
|
- !ruby/object:Gem::Dependency
|
29
71
|
name: rake
|
30
72
|
requirement: !ruby/object:Gem::Requirement
|
@@ -40,19 +82,19 @@ dependencies:
|
|
40
82
|
- !ruby/object:Gem::Version
|
41
83
|
version: '0'
|
42
84
|
- !ruby/object:Gem::Dependency
|
43
|
-
name:
|
85
|
+
name: coverage-kit
|
44
86
|
requirement: !ruby/object:Gem::Requirement
|
45
87
|
requirements:
|
46
88
|
- - ">="
|
47
89
|
- !ruby/object:Gem::Version
|
48
|
-
version: '0
|
90
|
+
version: '0'
|
49
91
|
type: :development
|
50
92
|
prerelease: false
|
51
93
|
version_requirements: !ruby/object:Gem::Requirement
|
52
94
|
requirements:
|
53
95
|
- - ">="
|
54
96
|
- !ruby/object:Gem::Version
|
55
|
-
version: '0
|
97
|
+
version: '0'
|
56
98
|
- !ruby/object:Gem::Dependency
|
57
99
|
name: rspec
|
58
100
|
requirement: !ruby/object:Gem::Requirement
|
@@ -81,29 +123,50 @@ dependencies:
|
|
81
123
|
- - ">="
|
82
124
|
- !ruby/object:Gem::Version
|
83
125
|
version: '3.5'
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: standardrb
|
128
|
+
requirement: !ruby/object:Gem::Requirement
|
129
|
+
requirements:
|
130
|
+
- - ">="
|
131
|
+
- !ruby/object:Gem::Version
|
132
|
+
version: '0'
|
133
|
+
type: :development
|
134
|
+
prerelease: false
|
135
|
+
version_requirements: !ruby/object:Gem::Requirement
|
136
|
+
requirements:
|
137
|
+
- - ">="
|
138
|
+
- !ruby/object:Gem::Version
|
139
|
+
version: '0'
|
140
|
+
- !ruby/object:Gem::Dependency
|
141
|
+
name: timecop
|
142
|
+
requirement: !ruby/object:Gem::Requirement
|
143
|
+
requirements:
|
144
|
+
- - ">="
|
145
|
+
- !ruby/object:Gem::Version
|
146
|
+
version: '0'
|
147
|
+
type: :development
|
148
|
+
prerelease: false
|
149
|
+
version_requirements: !ruby/object:Gem::Requirement
|
150
|
+
requirements:
|
151
|
+
- - ">="
|
152
|
+
- !ruby/object:Gem::Version
|
153
|
+
version: '0'
|
84
154
|
description: Use Redis to record the data migration status
|
85
155
|
email: support@travellink.com.au
|
86
|
-
executables:
|
156
|
+
executables:
|
157
|
+
- synchronised-migration
|
87
158
|
extensions: []
|
88
159
|
extra_rdoc_files: []
|
89
160
|
files:
|
90
|
-
- ".gitignore"
|
91
|
-
- ".ruby-version"
|
92
|
-
- ".travis.yml"
|
93
161
|
- CHANGELOG.md
|
94
|
-
- Gemfile
|
95
|
-
- LICENSE
|
96
162
|
- README.md
|
97
|
-
-
|
163
|
+
- exe/synchronised-migration
|
98
164
|
- lib/synchronised_migration.rb
|
165
|
+
- lib/synchronised_migration/commands.rb
|
166
|
+
- lib/synchronised_migration/configuration.rb
|
99
167
|
- lib/synchronised_migration/main.rb
|
100
|
-
- lib/synchronised_migration/railtie.rb
|
101
168
|
- lib/synchronised_migration/result.rb
|
102
169
|
- lib/synchronised_migration/version.rb
|
103
|
-
- lib/tasks/synchronised_migration.rake
|
104
|
-
- spec/spec_helper.rb
|
105
|
-
- spec/synchronised_migration/main_spec.rb
|
106
|
-
- synchronised_migration.gemspec
|
107
170
|
homepage: https://github.com/sealink/synchronised-migration-rb
|
108
171
|
licenses: []
|
109
172
|
metadata: {}
|
@@ -115,17 +178,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
115
178
|
requirements:
|
116
179
|
- - ">="
|
117
180
|
- !ruby/object:Gem::Version
|
118
|
-
version:
|
181
|
+
version: 2.6.0
|
119
182
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
120
183
|
requirements:
|
121
184
|
- - ">="
|
122
185
|
- !ruby/object:Gem::Version
|
123
186
|
version: '0'
|
124
187
|
requirements: []
|
125
|
-
rubygems_version: 3.
|
188
|
+
rubygems_version: 3.3.3
|
126
189
|
signing_key:
|
127
190
|
specification_version: 4
|
128
191
|
summary: For deploying to multiple instances simultaneously
|
129
|
-
test_files:
|
130
|
-
- spec/spec_helper.rb
|
131
|
-
- spec/synchronised_migration/main_spec.rb
|
192
|
+
test_files: []
|
data/.gitignore
DELETED
data/.ruby-version
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
2.6.3
|
data/.travis.yml
DELETED
data/Gemfile
DELETED
data/LICENSE
DELETED
@@ -1,21 +0,0 @@
|
|
1
|
-
MIT License
|
2
|
-
|
3
|
-
Copyright (c) 2017 SeaLink Travel Group Limited
|
4
|
-
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
-
of this software and associated documentation files (the "Software"), to deal
|
7
|
-
in the Software without restriction, including without limitation the rights
|
8
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
-
copies of the Software, and to permit persons to whom the Software is
|
10
|
-
furnished to do so, subject to the following conditions:
|
11
|
-
|
12
|
-
The above copyright notice and this permission notice shall be included in all
|
13
|
-
copies or substantial portions of the Software.
|
14
|
-
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
-
SOFTWARE.
|
data/Rakefile
DELETED
data/spec/spec_helper.rb
DELETED
@@ -1,164 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
require 'synchronised_migration/main'
|
3
|
-
|
4
|
-
describe SynchronisedMigration::Main do
|
5
|
-
subject { described_class }
|
6
|
-
let(:result) { subject.call }
|
7
|
-
|
8
|
-
context 'when the prerequisites are meet' do
|
9
|
-
let(:redis) { double }
|
10
|
-
let(:redlock) { double }
|
11
|
-
let(:fail_marker_value) { nil }
|
12
|
-
let(:success_marker_value) { nil }
|
13
|
-
let(:version_suffix) { 'bork' }
|
14
|
-
let(:set_version_suffix) { ENV['REDLOCK_VERSION_SUFFIX'] = version_suffix }
|
15
|
-
let(:time_value) { double(to_i: 123456789) }
|
16
|
-
|
17
|
-
before do
|
18
|
-
set_version_suffix
|
19
|
-
|
20
|
-
subject.instance.instance_variable_set :@redis, nil
|
21
|
-
subject.instance.instance_variable_set :@redlock, nil
|
22
|
-
|
23
|
-
allow(subject.instance).to receive(:execute).and_call_original
|
24
|
-
allow(subject.instance).to receive(:migrate).and_call_original
|
25
|
-
|
26
|
-
allow(Redis).to receive(:new).and_return(redis)
|
27
|
-
allow(redis).to receive(:get) { |key|
|
28
|
-
case key
|
29
|
-
when "migration-failed-#{version_suffix}"
|
30
|
-
fail_marker_value
|
31
|
-
when "migration-failed"
|
32
|
-
fail_marker_value
|
33
|
-
when "migration-success-#{version_suffix}"
|
34
|
-
success_marker_value
|
35
|
-
when "migration-success"
|
36
|
-
success_marker_value
|
37
|
-
else
|
38
|
-
raise "invalid key for redis get: #{key}"
|
39
|
-
end
|
40
|
-
}
|
41
|
-
allow(redis).to receive(:set)
|
42
|
-
allow(redis).to receive(:del)
|
43
|
-
|
44
|
-
allow(Redlock::Client).to receive(:new).and_return(redlock)
|
45
|
-
allow(redlock).to receive(:lock!) { |lock_key, timeout, &block| block.call }
|
46
|
-
|
47
|
-
allow(Time).to receive(:now).and_return(time_value)
|
48
|
-
|
49
|
-
allow(Kernel).to receive(:system).and_wrap_original { |method, *args|
|
50
|
-
next if args == [ 'bin/launch/migrate' ]
|
51
|
-
method.call *args
|
52
|
-
}
|
53
|
-
|
54
|
-
allow(Bundler).to receive(:with_original_env).and_call_original
|
55
|
-
|
56
|
-
SynchronisedMigration.configure do |config|
|
57
|
-
config.host = 'example.com'
|
58
|
-
config.port = 6379
|
59
|
-
config.db = 0
|
60
|
-
end
|
61
|
-
end
|
62
|
-
|
63
|
-
context 'in the happy path' do
|
64
|
-
it 'executes the migration successfully' do
|
65
|
-
expect(result).to be_success
|
66
|
-
expect(redlock).to have_received(:lock!)
|
67
|
-
expect(redis).to have_received(:get).with('migration-failed-bork')
|
68
|
-
expect(redis).to have_received(:set).with('migration-failed-bork', 123456789, ttl: 3600)
|
69
|
-
expect(redis).to have_received(:set).with('migration-success-bork', 123456789, ttl: 3600*24*30)
|
70
|
-
expect(Kernel).to have_received(:system)
|
71
|
-
expect(Bundler).not_to have_received(:with_original_env)
|
72
|
-
expect(redis).to have_received(:del).with('migration-failed-bork')
|
73
|
-
end
|
74
|
-
|
75
|
-
context 'and migration completed previously' do
|
76
|
-
let(:success_marker_value) { '1' }
|
77
|
-
it 'contines without executing' do
|
78
|
-
expect(result).to be_success
|
79
|
-
expect(redlock).not_to have_received(:lock!)
|
80
|
-
end
|
81
|
-
end
|
82
|
-
|
83
|
-
context 'executing in lock waiter' do
|
84
|
-
let(:result2) { subject.call }
|
85
|
-
|
86
|
-
before do
|
87
|
-
# Note: bypasses the first success flag check so it can enter lock.
|
88
|
-
expect(result).to be_success
|
89
|
-
allow(redis).to receive(:get).and_return(nil, '1')
|
90
|
-
end
|
91
|
-
|
92
|
-
it 'early exits and does not execute again', :aggregate_failures do
|
93
|
-
expect(result2).to be_success
|
94
|
-
expect(redlock).to have_received(:lock!).exactly(2).times
|
95
|
-
expect(Kernel).to have_received(:system).exactly(1).times
|
96
|
-
expect(subject.instance).to have_received(:execute).exactly(2).times
|
97
|
-
expect(subject.instance).to have_received(:migrate).exactly(1).times
|
98
|
-
end
|
99
|
-
end
|
100
|
-
end
|
101
|
-
|
102
|
-
context 'when require a clean Bundler environment' do
|
103
|
-
before do
|
104
|
-
allow(ENV).to receive(:fetch).and_call_original
|
105
|
-
allow(ENV).to receive(:fetch).with('WITH_CLEAN_BUNDLER_ENV', '').and_return('1')
|
106
|
-
end
|
107
|
-
|
108
|
-
it 'executes it with a clean Bundler environment' do
|
109
|
-
expect(result).to be_success
|
110
|
-
expect(Kernel).to have_received(:system)
|
111
|
-
expect(Bundler).to have_received(:with_original_env)
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
|
-
context 'after a deployment failed previously' do
|
116
|
-
let(:fail_marker_value) { '1' }
|
117
|
-
|
118
|
-
it "doesn't execute the migration" do
|
119
|
-
expect(result).not_to be_success
|
120
|
-
expect(Kernel).not_to have_received(:system)
|
121
|
-
end
|
122
|
-
end
|
123
|
-
|
124
|
-
context 'when the task crashed' do
|
125
|
-
before do
|
126
|
-
allow_any_instance_of(Process::Status).to receive(:success?).and_return(false)
|
127
|
-
end
|
128
|
-
|
129
|
-
it 'marks the failure in Redis' do
|
130
|
-
expect(result).not_to be_success
|
131
|
-
expect(redis).to have_received(:set).with('migration-failed-bork', 123456789, ttl: 3600)
|
132
|
-
expect(redis).not_to have_received(:del)
|
133
|
-
end
|
134
|
-
end
|
135
|
-
|
136
|
-
context 'without version suffix' do
|
137
|
-
let(:set_version_suffix) { ENV.delete 'REDLOCK_VERSION_SUFFIX' }
|
138
|
-
|
139
|
-
context 'in the happy path' do
|
140
|
-
it 'executes the migration successfully' do
|
141
|
-
expect(result).to be_success
|
142
|
-
expect(redlock).to have_received(:lock!)
|
143
|
-
expect(redis).to have_received(:get).with('migration-failed')
|
144
|
-
expect(redis).to have_received(:set).with('migration-failed', 123456789, ttl: 3600)
|
145
|
-
expect(Kernel).to have_received(:system)
|
146
|
-
expect(Bundler).not_to have_received(:with_original_env)
|
147
|
-
expect(redis).to have_received(:del).with('migration-failed')
|
148
|
-
end
|
149
|
-
end
|
150
|
-
|
151
|
-
context 'when the task crashed' do
|
152
|
-
before do
|
153
|
-
allow_any_instance_of(Process::Status).to receive(:success?).and_return(false)
|
154
|
-
end
|
155
|
-
|
156
|
-
it 'marks the failure in Redis' do
|
157
|
-
expect(result).not_to be_success
|
158
|
-
expect(redis).to have_received(:set).with('migration-failed', 123456789, ttl: 3600)
|
159
|
-
expect(redis).not_to have_received(:del)
|
160
|
-
end
|
161
|
-
end
|
162
|
-
end
|
163
|
-
end
|
164
|
-
end
|
@@ -1,24 +0,0 @@
|
|
1
|
-
lib = File.expand_path('../lib', __FILE__)
|
2
|
-
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
-
require 'synchronised_migration/version'
|
4
|
-
|
5
|
-
Gem::Specification.new do |spec|
|
6
|
-
spec.name = 'synchronised_migration'
|
7
|
-
spec.version = SynchronisedMigration::VERSION
|
8
|
-
spec.authors = ['Alvin Yim', 'Stefan Cooper']
|
9
|
-
spec.email = 'support@travellink.com.au'
|
10
|
-
spec.description = 'Use Redis to record the data migration status'
|
11
|
-
spec.summary = 'For deploying to multiple instances simultaneously'
|
12
|
-
spec.homepage = 'https://github.com/sealink/synchronised-migration-rb'
|
13
|
-
|
14
|
-
spec.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
|
15
|
-
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
16
|
-
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
17
|
-
spec.require_paths = ['lib']
|
18
|
-
|
19
|
-
spec.add_dependency 'redlock', '>= 0.2'
|
20
|
-
spec.add_development_dependency 'rake'
|
21
|
-
spec.add_development_dependency 'simplecov-rcov', '>= 0.2'
|
22
|
-
spec.add_development_dependency 'rspec', '>= 3.6'
|
23
|
-
spec.add_development_dependency 'pry-byebug', '>= 3.5'
|
24
|
-
end
|