cassanity 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +2 -1
- data/.travis.yml +1 -0
- data/Gemfile +3 -0
- data/Guardfile +0 -2
- data/README.md +11 -0
- data/doc/Instrumentation.md +40 -0
- data/doc/Migrations.md +132 -0
- data/examples/keyspaces.rb +11 -7
- data/lib/cassanity/argument_generators/column_family_delete.rb +1 -1
- data/lib/cassanity/argument_generators/columns.rb +33 -0
- data/lib/cassanity/argument_generators/where_clause.rb +1 -1
- data/lib/cassanity/client.rb +3 -1
- data/lib/cassanity/column.rb +48 -0
- data/lib/cassanity/column_family.rb +21 -2
- data/lib/cassanity/connection.rb +4 -8
- data/lib/cassanity/error.rb +18 -11
- data/lib/cassanity/executors/cassandra_cql.rb +79 -50
- data/lib/cassanity/instrumentation/log_subscriber.rb +4 -5
- data/lib/cassanity/instrumentation/metriks.rb +6 -0
- data/lib/cassanity/instrumentation/metriks_subscriber.rb +16 -0
- data/lib/cassanity/instrumentation/statsd.rb +6 -0
- data/lib/cassanity/instrumentation/statsd_subscriber.rb +22 -0
- data/lib/cassanity/instrumentation/subscriber.rb +58 -0
- data/lib/cassanity/instrumenters/memory.rb +0 -1
- data/lib/cassanity/keyspace.rb +10 -8
- data/lib/cassanity/migration.rb +125 -0
- data/lib/cassanity/migration_proxy.rb +76 -0
- data/lib/cassanity/migrator.rb +154 -0
- data/lib/cassanity/result_transformers/column_families.rb +20 -0
- data/lib/cassanity/result_transformers/columns.rb +21 -0
- data/lib/cassanity/result_transformers/keyspaces.rb +21 -0
- data/lib/cassanity/result_transformers/mirror.rb +1 -1
- data/lib/cassanity/result_transformers/result_to_array.rb +1 -1
- data/lib/cassanity/retry_strategies/exponential_backoff.rb +43 -0
- data/lib/cassanity/retry_strategies/retry_n_times.rb +29 -0
- data/lib/cassanity/retry_strategies/retry_strategy.rb +35 -0
- data/lib/cassanity/version.rb +1 -1
- data/spec/helper.rb +8 -0
- data/spec/integration/cassanity/column_family_spec.rb +36 -25
- data/spec/integration/cassanity/connection_spec.rb +11 -11
- data/spec/integration/cassanity/fixtures/migrations/20130224135000_create_users.rb +17 -0
- data/spec/integration/cassanity/fixtures/migrations/20130225135002_create_apps.rb +15 -0
- data/spec/integration/cassanity/fixtures/migrations/20130226135004_add_username_to_users.rb +9 -0
- data/spec/integration/cassanity/instrumentation/log_subscriber_spec.rb +71 -0
- data/spec/integration/cassanity/instrumentation/metriks_subscriber_spec.rb +48 -0
- data/spec/integration/cassanity/instrumentation/statsd_subscriber_spec.rb +58 -0
- data/spec/integration/cassanity/keyspace_spec.rb +21 -21
- data/spec/integration/cassanity/migration_spec.rb +157 -0
- data/spec/integration/cassanity/migrator_spec.rb +212 -0
- data/spec/support/cassanity_helpers.rb +21 -17
- data/spec/support/fake_udp_socket.rb +27 -0
- data/spec/unit/cassanity/argument_generators/batch_spec.rb +5 -5
- data/spec/unit/cassanity/argument_generators/column_family_delete_spec.rb +20 -6
- data/spec/unit/cassanity/argument_generators/column_family_update_spec.rb +6 -6
- data/spec/unit/cassanity/argument_generators/columns_spec.rb +45 -0
- data/spec/unit/cassanity/argument_generators/keyspace_create_spec.rb +1 -1
- data/spec/unit/cassanity/argument_generators/keyspace_drop_spec.rb +1 -1
- data/spec/unit/cassanity/argument_generators/keyspace_use_spec.rb +1 -1
- data/spec/unit/cassanity/argument_generators/where_clause_spec.rb +2 -2
- data/spec/unit/cassanity/client_spec.rb +10 -3
- data/spec/unit/cassanity/column_family_spec.rb +20 -3
- data/spec/unit/cassanity/column_spec.rb +76 -0
- data/spec/unit/cassanity/connection_spec.rb +1 -1
- data/spec/unit/cassanity/error_spec.rb +7 -2
- data/spec/unit/cassanity/executors/cassandra_cql_spec.rb +76 -23
- data/spec/unit/cassanity/keyspace_spec.rb +38 -13
- data/spec/unit/cassanity/migration_proxy_spec.rb +81 -0
- data/spec/unit/cassanity/migration_spec.rb +12 -0
- data/spec/unit/cassanity/migrator_spec.rb +20 -0
- data/spec/unit/cassanity/retry_strategies/exponential_backoff_spec.rb +37 -0
- data/spec/unit/cassanity/retry_strategies/retry_n_times_spec.rb +47 -0
- data/spec/unit/cassanity/retry_strategies/retry_strategy_spec.rb +27 -0
- metadata +56 -4
@@ -0,0 +1,125 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
|
3
|
+
module Cassanity
|
4
|
+
class Migration
|
5
|
+
extend Forwardable
|
6
|
+
|
7
|
+
# Private: The migrator that is running the migration.
|
8
|
+
attr_reader :migrator
|
9
|
+
|
10
|
+
# Private: Delegates keyspace to migrator.
|
11
|
+
def_delegator :@migrator, :keyspace
|
12
|
+
|
13
|
+
# Public: Get new instance of a migration.
|
14
|
+
#
|
15
|
+
# migrator - The Cassanity::Migrator instance that is running the show.
|
16
|
+
def initialize(migrator)
|
17
|
+
@migrator = migrator
|
18
|
+
end
|
19
|
+
|
20
|
+
# Public: Override in subclass.
|
21
|
+
def up
|
22
|
+
end
|
23
|
+
|
24
|
+
# Public: Override in subclass.
|
25
|
+
def down
|
26
|
+
end
|
27
|
+
|
28
|
+
# Public: Create a column family.
|
29
|
+
#
|
30
|
+
# column_family_name - The String or Symbol name of the column family.
|
31
|
+
# args - The Hash of arguments. See ColumnFamily#create for available args.
|
32
|
+
#
|
33
|
+
# Returns nothing.
|
34
|
+
def create_column_family(column_family_name, schema)
|
35
|
+
keyspace.column_family(column_family_name, schema: schema).create
|
36
|
+
end
|
37
|
+
alias_method :add_column_family, :create_column_family
|
38
|
+
alias_method :create_table, :create_column_family
|
39
|
+
alias_method :add_table, :create_column_family
|
40
|
+
|
41
|
+
# Public: Drop a column family.
|
42
|
+
#
|
43
|
+
# column_family_name - The String or Symbol name of the column family.
|
44
|
+
#
|
45
|
+
# Returns nothing.
|
46
|
+
def drop_column_family(column_family_name)
|
47
|
+
keyspace[column_family_name].drop
|
48
|
+
end
|
49
|
+
alias_method :drop_table, :drop_column_family
|
50
|
+
|
51
|
+
# Public: Add a column to a column family.
|
52
|
+
#
|
53
|
+
# column_family_name - The String or Symbol name of the column family.
|
54
|
+
# column_name - The String or Symbol name of the column to index.
|
55
|
+
# type - The String or Symbol CQL data type for the column.
|
56
|
+
#
|
57
|
+
# Returns nothing.
|
58
|
+
def add_column(column_family_name, column_name, type)
|
59
|
+
keyspace[column_family_name].alter(add: {column_name => type})
|
60
|
+
end
|
61
|
+
alias_method :create_column, :add_column
|
62
|
+
|
63
|
+
# Public: Drop a column from a column family.
|
64
|
+
#
|
65
|
+
# column_family_name - The String or Symbol name of the column family.
|
66
|
+
# column_name - The String or Symbol name of the column to index.
|
67
|
+
#
|
68
|
+
# Returns nothing.
|
69
|
+
def drop_column(column_family_name, column_name)
|
70
|
+
keyspace[column_family_name].alter(drop: column_name)
|
71
|
+
end
|
72
|
+
|
73
|
+
# Public: Alter a column family.
|
74
|
+
#
|
75
|
+
# column_family_name - The String or Symbol name of the column family.
|
76
|
+
# args - The Hash of arguments. See ColumnFamily#alter for available args.
|
77
|
+
#
|
78
|
+
# Returns nothing.
|
79
|
+
def alter_column_family(column_family_name, args = {})
|
80
|
+
keyspace[column_family_name].alter(args)
|
81
|
+
end
|
82
|
+
alias_method :alter_table, :alter_column_family
|
83
|
+
|
84
|
+
# Public: Create an index on a column for a column family.
|
85
|
+
#
|
86
|
+
# column_family_name - The String or Symbol name of the column family.
|
87
|
+
# column_name - The String or Symbol name of the column to index.
|
88
|
+
# options - The Hash of options.
|
89
|
+
# :name - The String or Symbol name of the index
|
90
|
+
# (defaults to column_name).
|
91
|
+
#
|
92
|
+
# Returns nothing.
|
93
|
+
def add_index(column_family_name, column_name, options = {})
|
94
|
+
index_args = {column_name: column_name}
|
95
|
+
index_args[:name] = options[:name] if options.key?(:name)
|
96
|
+
|
97
|
+
keyspace[column_family_name].create_index(index_args)
|
98
|
+
end
|
99
|
+
alias_method :create_index, :add_index
|
100
|
+
|
101
|
+
# Public: Drop an index by name for a column family.
|
102
|
+
#
|
103
|
+
# column_family_name - The String or Symbol name of the column family.
|
104
|
+
# name - The String or Symbol name of the index.
|
105
|
+
#
|
106
|
+
# Returns nothing.
|
107
|
+
def drop_index(column_family_name, index_name)
|
108
|
+
keyspace[column_family_name].drop_index(name: index_name)
|
109
|
+
end
|
110
|
+
|
111
|
+
# Public: Spit something to the log with timing.
|
112
|
+
def say_with_time(message)
|
113
|
+
say message
|
114
|
+
start = Time.now
|
115
|
+
result = yield
|
116
|
+
duration = (Time.now - start).round(3)
|
117
|
+
@migrator.log " -> #{duration}s"
|
118
|
+
end
|
119
|
+
|
120
|
+
# Public: Spit something to the log.
|
121
|
+
def say(message)
|
122
|
+
@migrator.log "-- #{message}"
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
require 'pathname'
|
2
|
+
|
3
|
+
module Cassanity
|
4
|
+
class MigrationProxy
|
5
|
+
include Comparable
|
6
|
+
|
7
|
+
# Public: The full path to the migration on disk.
|
8
|
+
attr_reader :path
|
9
|
+
|
10
|
+
# Public: The version of the migration.
|
11
|
+
attr_reader :version
|
12
|
+
|
13
|
+
# Public: The name of the migration.
|
14
|
+
attr_reader :name
|
15
|
+
|
16
|
+
def initialize(path)
|
17
|
+
raise ArgumentError, "path cannot be nil" if path.nil?
|
18
|
+
|
19
|
+
basename = File.basename(path, '.rb')
|
20
|
+
version, name = basename.split('_', 2)
|
21
|
+
|
22
|
+
raise ArgumentError, "version cannot be nil" if version.nil?
|
23
|
+
raise ArgumentError, "name cannot be nil" if name.nil?
|
24
|
+
|
25
|
+
@path = Pathname(path)
|
26
|
+
@version = version.to_i
|
27
|
+
@name = name
|
28
|
+
end
|
29
|
+
|
30
|
+
def up(migrator)
|
31
|
+
log(migrator) { build_migration(migrator).up }
|
32
|
+
end
|
33
|
+
|
34
|
+
def down(migrator)
|
35
|
+
log(migrator) { build_migration(migrator).down }
|
36
|
+
end
|
37
|
+
|
38
|
+
def log(migrator)
|
39
|
+
migrator.log "== #{@name}: migrating ".ljust(80, "=")
|
40
|
+
start = Time.now
|
41
|
+
result = yield
|
42
|
+
duration = (Time.now - start).round(3)
|
43
|
+
migrator.log "== #{@name}: migrated (#{duration}s) ".ljust(80, "=")
|
44
|
+
result
|
45
|
+
end
|
46
|
+
|
47
|
+
def build_migration(migrator)
|
48
|
+
migration_class.new(migrator)
|
49
|
+
end
|
50
|
+
|
51
|
+
def migration_class
|
52
|
+
@migration_class ||= begin
|
53
|
+
require path
|
54
|
+
# TODO: handle constant not found
|
55
|
+
Kernel.const_get(constant)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def constant
|
60
|
+
name.split('_').map { |word| word.capitalize }.join('')
|
61
|
+
end
|
62
|
+
|
63
|
+
def hash
|
64
|
+
path.hash
|
65
|
+
end
|
66
|
+
|
67
|
+
def eql?(other)
|
68
|
+
self.class.eql?(other.class) && path == other.path
|
69
|
+
end
|
70
|
+
alias_method :==, :eql?
|
71
|
+
|
72
|
+
def <=>(other)
|
73
|
+
@path <=> other.path
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
@@ -0,0 +1,154 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require 'pathname'
|
3
|
+
require 'cassanity/migration_proxy'
|
4
|
+
require 'cassanity/migration'
|
5
|
+
|
6
|
+
module Cassanity
|
7
|
+
class Migrator
|
8
|
+
SupportedDirections = [:up, :down]
|
9
|
+
|
10
|
+
# Public: The keyspace all migrations apply to.
|
11
|
+
attr_reader :keyspace
|
12
|
+
|
13
|
+
# Public: The path to all the migrations.
|
14
|
+
attr_reader :migrations_path
|
15
|
+
|
16
|
+
# Public: Where to spit all the logging related to migrations.
|
17
|
+
attr_reader :logger
|
18
|
+
|
19
|
+
def initialize(keyspace, migrations_path, options = {})
|
20
|
+
@keyspace = keyspace
|
21
|
+
@migrations_path = Pathname(migrations_path)
|
22
|
+
@logger = options[:logger] || default_logger
|
23
|
+
end
|
24
|
+
|
25
|
+
# Public: Runs all pending migrations in order.
|
26
|
+
def migrate
|
27
|
+
run_migrations pending_migrations, :up
|
28
|
+
end
|
29
|
+
|
30
|
+
# Public: Migrates to a version using a direction.
|
31
|
+
#
|
32
|
+
# version - The String or Integer version to migrate to.
|
33
|
+
# direction - The String or Symbol direction you would like to migrate.
|
34
|
+
def migrate_to(version, direction = :up)
|
35
|
+
version = version.to_i
|
36
|
+
direction = direction.to_sym
|
37
|
+
assert_valid_direction(direction)
|
38
|
+
|
39
|
+
migrations = case direction
|
40
|
+
when :up
|
41
|
+
pending_migrations.select { |migration| migration.version <= version }
|
42
|
+
when :down
|
43
|
+
performed_migrations.select { |migration| migration.version > version }
|
44
|
+
else
|
45
|
+
[]
|
46
|
+
end
|
47
|
+
|
48
|
+
run_migrations migrations, direction
|
49
|
+
end
|
50
|
+
|
51
|
+
# Public: Marks a migration as migrated.
|
52
|
+
def migrated_up(migration)
|
53
|
+
column_family.insert({
|
54
|
+
data: {
|
55
|
+
version: migration.version,
|
56
|
+
name: migration.name,
|
57
|
+
migrated_at: Time.now.utc,
|
58
|
+
},
|
59
|
+
})
|
60
|
+
end
|
61
|
+
|
62
|
+
# Public: Marks a migration as not run.
|
63
|
+
def migrated_down(migration)
|
64
|
+
column_family.delete({
|
65
|
+
where: {
|
66
|
+
version: migration.version,
|
67
|
+
name: migration.name,
|
68
|
+
},
|
69
|
+
})
|
70
|
+
end
|
71
|
+
|
72
|
+
# Public: An array of all migrations.
|
73
|
+
def migrations
|
74
|
+
@migrations ||= begin
|
75
|
+
paths = Dir["#{migrations_path}/*.rb"]
|
76
|
+
migrations = paths.map { |path| MigrationProxy.new(path) }
|
77
|
+
migrations.sort
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
# Public: An array of the migrations that have been performed.
|
82
|
+
def performed_migrations
|
83
|
+
rows = column_family.select
|
84
|
+
rows.map { |row|
|
85
|
+
path = migrations_path.join("#{row['version']}_#{row['name']}.rb")
|
86
|
+
MigrationProxy.new(path)
|
87
|
+
}.sort
|
88
|
+
end
|
89
|
+
|
90
|
+
# Public: An array of the migrations that have not been performed.
|
91
|
+
def pending_migrations
|
92
|
+
(migrations - performed_migrations).sort
|
93
|
+
end
|
94
|
+
|
95
|
+
# Internal: Log a message.
|
96
|
+
def log(message)
|
97
|
+
@logger.info message
|
98
|
+
end
|
99
|
+
|
100
|
+
# Private
|
101
|
+
def run_migrations(migrations, direction)
|
102
|
+
migrations = migrations.sort
|
103
|
+
|
104
|
+
case direction
|
105
|
+
when :up
|
106
|
+
migrations.each do |migration|
|
107
|
+
migration.up(self)
|
108
|
+
migrated_up(migration)
|
109
|
+
end
|
110
|
+
when :down
|
111
|
+
migrations = migrations.reverse
|
112
|
+
migrations.each do |migration|
|
113
|
+
migration.down(self)
|
114
|
+
migrated_down(migration)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
{performed: migrations}
|
119
|
+
end
|
120
|
+
|
121
|
+
# Private: The column family storing all
|
122
|
+
# migration information.
|
123
|
+
def column_family
|
124
|
+
@column_family ||= begin
|
125
|
+
column_family = keyspace.column_family({
|
126
|
+
name: :migrations,
|
127
|
+
schema: {
|
128
|
+
primary_key: [:version, :name],
|
129
|
+
columns: {
|
130
|
+
version: :text,
|
131
|
+
name: :text,
|
132
|
+
migrated_at: :timestamp,
|
133
|
+
},
|
134
|
+
},
|
135
|
+
})
|
136
|
+
column_family.create unless column_family.exists?
|
137
|
+
column_family
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
# Private
|
142
|
+
def default_logger
|
143
|
+
logger = Logger.new(STDOUT)
|
144
|
+
logger.formatter = proc { |_, _, _, msg| "#{msg}\n" }
|
145
|
+
logger
|
146
|
+
end
|
147
|
+
|
148
|
+
def assert_valid_direction(direction)
|
149
|
+
unless SupportedDirections.include?(direction)
|
150
|
+
raise ArgumentError, "#{direction.inspect} is not a valid migration direction"
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'cassanity/column_family'
|
2
|
+
|
3
|
+
module Cassanity
|
4
|
+
module ResultTransformers
|
5
|
+
class ColumnFamilies
|
6
|
+
|
7
|
+
# Internal: Turns result into Array of column families.
|
8
|
+
def call(result, args = {})
|
9
|
+
column_families = []
|
10
|
+
result.fetch_hash do |row|
|
11
|
+
column_families << ColumnFamily.new({
|
12
|
+
name: row['columnfamily'] || row['columnfamily_name'],
|
13
|
+
keyspace: args[:keyspace],
|
14
|
+
})
|
15
|
+
end
|
16
|
+
column_families
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'cassanity/column'
|
2
|
+
|
3
|
+
module Cassanity
|
4
|
+
module ResultTransformers
|
5
|
+
class Columns
|
6
|
+
|
7
|
+
# Internal: Turns result into Array of column families.
|
8
|
+
def call(result, args = {})
|
9
|
+
columns = []
|
10
|
+
result.fetch_hash do |row|
|
11
|
+
columns << Column.new({
|
12
|
+
name: row['column'],
|
13
|
+
type: row['validator'],
|
14
|
+
column_family: args[:column_family],
|
15
|
+
})
|
16
|
+
end
|
17
|
+
columns
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'cassanity/column_family'
|
2
|
+
|
3
|
+
module Cassanity
|
4
|
+
module ResultTransformers
|
5
|
+
class Keyspaces
|
6
|
+
|
7
|
+
# Internal: Turns result into Array of keyspaces.
|
8
|
+
def call(result, args = {})
|
9
|
+
keyspaces = []
|
10
|
+
result.fetch_hash do |row|
|
11
|
+
name = row['name'] || row['keyspace'] || row['keyspace_name']
|
12
|
+
keyspaces << Keyspace.new({
|
13
|
+
name: name,
|
14
|
+
executor: args[:executor],
|
15
|
+
})
|
16
|
+
end
|
17
|
+
keyspaces
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require 'cassanity/retry_strategies/retry_strategy'
|
2
|
+
|
3
|
+
module Cassanity
|
4
|
+
module RetryStrategies
|
5
|
+
class ExponentialBackoff < RetryStrategy
|
6
|
+
ForeverSentinel = :forever
|
7
|
+
|
8
|
+
# Private: Taken from https://github.com/twitter/kestrel-client's
|
9
|
+
# blocking client.
|
10
|
+
SleepTimes = [[0] * 1, [0.01] * 2, [0.1] * 2, [0.5] * 2, [1.0] * 1].flatten
|
11
|
+
|
12
|
+
# Private: the maxmimum number of times to retry or -1 to try forever.
|
13
|
+
attr_reader :retries
|
14
|
+
|
15
|
+
# Public: Initialize the retry strategy.
|
16
|
+
#
|
17
|
+
# args - The Hash of options.
|
18
|
+
# :retries - the maximum number of times to retry (default: forever)
|
19
|
+
def initialize(args = {})
|
20
|
+
# The default behavior is to retry forever.
|
21
|
+
@retries = args[:retries] || ForeverSentinel
|
22
|
+
end
|
23
|
+
|
24
|
+
def fail(attempts, error)
|
25
|
+
if @retries != ForeverSentinel && attempts > @retries
|
26
|
+
raise error
|
27
|
+
end
|
28
|
+
sleep_for_count(attempts)
|
29
|
+
end
|
30
|
+
|
31
|
+
# Private: sleep a randomized amount of time from the SleepTimes
|
32
|
+
# mostly-exponential distribution.
|
33
|
+
#
|
34
|
+
# count - the index into the distribution to pull the base sleep time from
|
35
|
+
def sleep_for_count(count)
|
36
|
+
base = SleepTimes[count] || SleepTimes.last
|
37
|
+
|
38
|
+
time = ((rand * base) + base) / 2
|
39
|
+
sleep time if time > 0
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|