datastax_rails 1.1.0.3 → 1.2.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.rdoc +13 -13
- data/Rakefile +1 -0
- data/config/schema.xml.erb +0 -1
- data/config/{solrconfig.xml → solrconfig.xml.erb} +1 -1
- data/lib/blankslate.rb +1 -1
- data/lib/datastax_rails/associations/collection_proxy.rb +6 -2
- data/lib/datastax_rails/attribute_assignment.rb +114 -0
- data/lib/datastax_rails/attribute_methods/definition.rb +8 -2
- data/lib/datastax_rails/attribute_methods/typecasting.rb +2 -5
- data/lib/datastax_rails/attribute_methods.rb +9 -7
- data/lib/datastax_rails/base.rb +127 -109
- data/lib/datastax_rails/callbacks.rb +11 -7
- data/lib/datastax_rails/cassandra_only_model.rb +27 -0
- data/lib/datastax_rails/collection.rb +3 -1
- data/lib/datastax_rails/cql/base.rb +4 -0
- data/lib/datastax_rails/cql/select.rb +12 -2
- data/lib/datastax_rails/identity/abstract_key_factory.rb +1 -0
- data/lib/datastax_rails/identity/custom_key_factory.rb +1 -0
- data/lib/datastax_rails/identity/natural_key_factory.rb +1 -0
- data/lib/datastax_rails/identity/uuid_key_factory.rb +4 -0
- data/lib/datastax_rails/identity.rb +2 -1
- data/lib/datastax_rails/inheritance.rb +61 -0
- data/lib/datastax_rails/payload_model.rb +2 -5
- data/lib/datastax_rails/persistence.rb +63 -20
- data/lib/datastax_rails/railtie.rb +5 -1
- data/lib/datastax_rails/relation/batches.rb +2 -2
- data/lib/datastax_rails/relation/facet_methods.rb +56 -5
- data/lib/datastax_rails/relation/finder_methods.rb +55 -1
- data/lib/datastax_rails/relation/search_methods.rb +103 -32
- data/lib/datastax_rails/relation/spawn_methods.rb +3 -1
- data/lib/datastax_rails/relation/stats_methods.rb +1 -1
- data/lib/datastax_rails/relation.rb +166 -30
- data/lib/datastax_rails/schema/cassandra.rb +165 -0
- data/lib/datastax_rails/schema/migrator.rb +85 -193
- data/lib/datastax_rails/schema/solr.rb +158 -0
- data/lib/datastax_rails/schema.rb +2 -30
- data/lib/datastax_rails/scoping/default.rb +142 -0
- data/lib/datastax_rails/scoping/named.rb +200 -0
- data/lib/datastax_rails/scoping.rb +106 -349
- data/lib/datastax_rails/tasks/ds.rake +41 -42
- data/lib/datastax_rails/types/array_type.rb +1 -1
- data/lib/datastax_rails/types/base_type.rb +2 -2
- data/lib/datastax_rails/types/binary_type.rb +1 -1
- data/lib/datastax_rails/types/boolean_type.rb +1 -1
- data/lib/datastax_rails/types/date_type.rb +1 -1
- data/lib/datastax_rails/types/float_type.rb +4 -4
- data/lib/datastax_rails/types/integer_type.rb +3 -3
- data/lib/datastax_rails/types/string_type.rb +1 -1
- data/lib/datastax_rails/types/text_type.rb +1 -1
- data/lib/datastax_rails/types/time_type.rb +3 -3
- data/lib/datastax_rails/validations/uniqueness.rb +1 -1
- data/lib/datastax_rails/version.rb +1 -1
- data/lib/datastax_rails/wide_storage_model.rb +44 -0
- data/lib/datastax_rails.rb +16 -18
- data/spec/datastax_rails/associations_spec.rb +7 -3
- data/spec/datastax_rails/attribute_methods_spec.rb +23 -0
- data/spec/datastax_rails/base_spec.rb +1 -6
- data/spec/datastax_rails/inheritance_spec.rb +41 -0
- data/spec/datastax_rails/persistence_spec.rb +13 -3
- data/spec/datastax_rails/relation/batches_spec.rb +1 -1
- data/spec/datastax_rails/relation/facet_methods_spec.rb +52 -0
- data/spec/datastax_rails/relation/finder_methods_spec.rb +22 -1
- data/spec/datastax_rails/relation/search_methods_spec.rb +51 -1
- data/spec/datastax_rails/relation_spec.rb +14 -3
- data/spec/datastax_rails/schema/migrator_spec.rb +92 -0
- data/spec/datastax_rails/schema/solr_spec.rb +34 -0
- data/spec/datastax_rails/scoping/default_spec.rb +17 -0
- data/spec/datastax_rails/types/float_type_spec.rb +5 -9
- data/spec/datastax_rails/types/integer_type_spec.rb +5 -9
- data/spec/datastax_rails/types/time_type_spec.rb +28 -0
- data/spec/datastax_rails/validations/uniqueness_spec.rb +3 -1
- data/spec/dummy/config/application.rb +1 -4
- data/spec/dummy/config/datastax.yml +1 -1
- data/spec/dummy/config/environments/test.rb +2 -0
- data/spec/dummy/config/solr/articles-schema.xml.erb +1 -0
- data/spec/dummy/db/test.sqlite3 +0 -0
- data/spec/dummy/log/development.log +2 -0
- data/spec/dummy/log/production.log +2 -0
- data/spec/dummy/log/test.log +523 -0
- data/spec/spec_helper.rb +11 -0
- data/spec/support/models.rb +14 -0
- metadata +66 -22
- data/lib/datastax_rails/log_subscriber.rb +0 -37
- data/lib/datastax_rails/migrations/migration.rb +0 -15
- data/lib/datastax_rails/migrations.rb +0 -36
- data/lib/datastax_rails/mocking.rb +0 -15
- data/lib/datastax_rails/schema/migration.rb +0 -106
- data/lib/datastax_rails/schema/migration_proxy.rb +0 -25
- data/lib/datastax_rails/tasks/column_family.rb +0 -329
- data/lib/datastax_rails/tasks/keyspace.rb +0 -57
- data/spec/support/connection_double.rb +0 -6
@@ -0,0 +1,165 @@
|
|
1
|
+
module DatastaxRails
|
2
|
+
module Schema
|
3
|
+
module Cassandra
|
4
|
+
# Check for missing columns or columns needing cassandra indexes
|
5
|
+
def check_missing_schema(model)
|
6
|
+
count = 0
|
7
|
+
model.attribute_definitions.each do |attribute, definition|
|
8
|
+
unless column_exists?(model.column_family.to_s, attribute.to_s)
|
9
|
+
count += 1
|
10
|
+
say "Adding column '#{attribute}'", :subitem
|
11
|
+
DatastaxRails::Cql::AlterColumnFamily.new(model.column_family).add(attribute => :text).execute
|
12
|
+
end
|
13
|
+
if(definition.indexed == :cassandra)
|
14
|
+
unless index_exists?(model.column_family.to_s, attribute.to_s)
|
15
|
+
if index_exists?(model.column_family.to_s, attribute.to_s)
|
16
|
+
count += 1
|
17
|
+
say "Dropping solr index on #{attribute.to_s}", :subitem
|
18
|
+
DatastaxRails::Cql::DropIndex.new(solr_index_cql_name(model.column_family.to_s, attribute.to_s)).execute
|
19
|
+
end
|
20
|
+
count += 1
|
21
|
+
say "Creating cassandra index on #{attribute.to_s}", :subitem
|
22
|
+
DatastaxRails::Cql::CreateIndex.new(cassandra_index_cql_name(model.column_family.to_s, attribute.to_s)).on(model.column_family.to_s).column(attribute.to_s).execute
|
23
|
+
end
|
24
|
+
elsif(definition.indexed == :both)
|
25
|
+
unless column_exists?(model.column_family.to_s, "__#{attribute.to_s}")
|
26
|
+
# Create and populate the new column
|
27
|
+
count += 1
|
28
|
+
say "Adding column '__#{attribute}'", :subitem
|
29
|
+
DatastaxRails::Cql::AlterColumnFamily.new(model.column_family).add("__#{attribute.to_s}" => model.attribute_definitions[attribute].coder.options[:cassandra_type]).execute
|
30
|
+
say "Populating column '__#{attribute}' (this might take a while)", :subitem
|
31
|
+
export = "echo \"copy #{model.column_family.to_s} (key, #{attribute.to_s}) TO 'dsr_export.csv';\" | cqlsh #{model.current_server}"
|
32
|
+
import = "echo \"copy #{model.column_family.to_s} (key, __#{attribute.to_s}) FROM 'dsr_export.csv';\" | cqlsh #{model.current_server}"
|
33
|
+
if system(export)
|
34
|
+
system(import)
|
35
|
+
else
|
36
|
+
@errors << "Looks like you don't have a working cqlsh command in your path.\nRun the following two commands from a server with cqlsh:\n\n#{export}\n#{import}"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
count += 1
|
40
|
+
say "Creating cassandra index on __#{attribute.to_s}", :subitem
|
41
|
+
DatastaxRails::Cql::CreateIndex.new(cassandra_index_cql_name(model.column_family.to_s, "__#{attribute.to_s}")).on(model.column_family.to_s).column("__#{attribute.to_s}").execute
|
42
|
+
end
|
43
|
+
end
|
44
|
+
count
|
45
|
+
end
|
46
|
+
|
47
|
+
# Creates a payload column family via CQL
|
48
|
+
def create_payload_column_family(model)
|
49
|
+
say "Creating Payload Column Family", :subitem
|
50
|
+
columns = {:chunk => :int, :payload => :text}
|
51
|
+
DatastaxRails::Cql::CreateColumnFamily.new(model.column_family).key_name(:digest).key_columns("digest, chunk").key_type(:text).columns(columns).with("COMPACT STORAGE").execute
|
52
|
+
end
|
53
|
+
|
54
|
+
# Creates a wide-storage column family via CQL
|
55
|
+
def create_wide_storage_column_family(model)
|
56
|
+
say "Creating Wide-Storage Column Family", :subitem
|
57
|
+
key_name = model.key_factory.attributes.join
|
58
|
+
cluster_by = model.cluster_by.keys.first
|
59
|
+
cluster_dir = model.cluster_by.values.first
|
60
|
+
key_columns = "#{key_name}, #{cluster_by}"
|
61
|
+
columns = {}
|
62
|
+
model.attribute_definitions.each {|k,v| columns[k] = v.coder.options[:cassandra_type] unless k.to_s == key_name}
|
63
|
+
DatastaxRails::Cql::CreateColumnFamily.new(model.column_family).key_name(key_name).key_columns(key_columns).key_type(:text).columns(columns).
|
64
|
+
with("CLUSTERING ORDER BY (#{cluster_by} #{cluster_dir.to_s.upcase})").execute
|
65
|
+
end
|
66
|
+
|
67
|
+
# Creates a regular cassandra-only column family via CQL
|
68
|
+
def create_cassandra_column_family(model)
|
69
|
+
say "Creating Cassandra-Only Column Family", :subitem
|
70
|
+
key_name = "key"
|
71
|
+
key_columns = "#{key_name}"
|
72
|
+
columns = {}
|
73
|
+
model.attribute_definitions.each {|k,v| columns[k] = v.coder.options[:cassandra_type]}
|
74
|
+
DatastaxRails::Cql::CreateColumnFamily.new(model.column_family).key_name(key_name).key_columns(key_columns).key_type(:text).columns(columns).execute
|
75
|
+
end
|
76
|
+
|
77
|
+
# Creates the named keyspace
|
78
|
+
def create_keyspace(keyspace, options = {})
|
79
|
+
opts = { :name => keyspace.to_s,
|
80
|
+
:strategy_class => 'org.apache.cassandra.locator.NetworkTopologyStrategy'}.with_indifferent_access.merge(options)
|
81
|
+
|
82
|
+
if(connection.keyspaces.collect(&:name).include?(keyspace.to_s))
|
83
|
+
say "Keyspace #{keyspace.to_s} already exists"
|
84
|
+
return false
|
85
|
+
else
|
86
|
+
cql = DatastaxRails::Cql::CreateKeyspace.new(opts.delete(:name))
|
87
|
+
cql.strategy_class(opts.delete(:strategy_class))
|
88
|
+
strategy_options = opts.delete('strategy_options')
|
89
|
+
cql.strategy_options(strategy_options.symbolize_keys)
|
90
|
+
say "Creating keyspace #{keyspace.to_s}"
|
91
|
+
cql.execute
|
92
|
+
return true
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def drop_keyspace
|
97
|
+
say "Dropping keyspace #{@keyspace.to_s}"
|
98
|
+
DatastaxRails::Cql::DropKeyspace.new(@keyspace.to_s).execute
|
99
|
+
end
|
100
|
+
|
101
|
+
# Checks the Cassandra system tables to see if the key column is named properly. This is
|
102
|
+
# a migration method to handle the fact that Solr used to create column families with "KEY"
|
103
|
+
# instead of the now default "key".
|
104
|
+
def check_key_name(cf)
|
105
|
+
count = 0
|
106
|
+
if(cf.respond_to?(:column_family))
|
107
|
+
cf = cf.column_family
|
108
|
+
end
|
109
|
+
klass = OpenStruct.new(:column_family => 'system.schema_columnfamilies', :default_consistency => 'QUORUM')
|
110
|
+
cql = DatastaxRails::Cql::ColumnFamily.new(klass)
|
111
|
+
results = CassandraCQL::Result.new(cql.select("key_alias, key_aliases").conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf).execute)
|
112
|
+
result = results.fetch
|
113
|
+
if(result && (result['key_alias'] == 'KEY' || result['key_aliases'].include?('KEY')) && (result['key_aliases'].blank? || !result['key_aliases'].include?('key')))
|
114
|
+
count += 1
|
115
|
+
say "Renaming KEY column", :subitem
|
116
|
+
DatastaxRails::Cql::AlterColumnFamily.new(cf).rename("KEY",'key').execute
|
117
|
+
end
|
118
|
+
count
|
119
|
+
end
|
120
|
+
|
121
|
+
# Computes the expected solr index name as reported by CQL.
|
122
|
+
def solr_index_cql_name(cf, column)
|
123
|
+
"#{@keyspace}_#{cf.to_s}_#{column.to_s}_index"
|
124
|
+
end
|
125
|
+
|
126
|
+
# Computes the expected cassandra index name as reported by CQL.
|
127
|
+
def cassandra_index_cql_name(cf, column)
|
128
|
+
"#{cf.to_s}_#{column.to_s}_idx"
|
129
|
+
end
|
130
|
+
|
131
|
+
# Checks the Cassandra system tables to see if a column family exists
|
132
|
+
def column_family_exists?(cf)
|
133
|
+
klass = OpenStruct.new(:column_family => 'system.schema_columnfamilies', :default_consistency => 'QUORUM')
|
134
|
+
cql = DatastaxRails::Cql::ColumnFamily.new(klass)
|
135
|
+
results = CassandraCQL::Result.new(cql.select("count(*)").conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf).execute)
|
136
|
+
results.fetch['count'] > 0
|
137
|
+
end
|
138
|
+
|
139
|
+
# Checks the Cassandra system tables to see if a column exists on a column family
|
140
|
+
def column_exists?(cf, col)
|
141
|
+
klass = OpenStruct.new(:column_family => 'system.schema_columns', :default_consistency => 'QUORUM')
|
142
|
+
cql = DatastaxRails::Cql::ColumnFamily.new(klass)
|
143
|
+
results = CassandraCQL::Result.new(cql.select("count(*)").conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf, 'column_name' => col).execute)
|
144
|
+
exists = results.fetch['count'] > 0
|
145
|
+
unless exists
|
146
|
+
# We need to check if it's part of the primary key (ugh)
|
147
|
+
klass = OpenStruct.new(:column_family => 'system.schema_columnfamilies', :default_consistency => 'QUORUM')
|
148
|
+
cql = DatastaxRails::Cql::ColumnFamily.new(klass)
|
149
|
+
results = CassandraCQL::Result.new(cql.select("column_aliases, key_aliases").conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf).execute)
|
150
|
+
row = results.fetch
|
151
|
+
exists = row['key_aliases'].include?(col.to_s) || row['column_aliases'].include?(col.to_s)
|
152
|
+
end
|
153
|
+
exists
|
154
|
+
end
|
155
|
+
|
156
|
+
# Checks the Cassandra system tables to see if an index exists on a column family
|
157
|
+
def index_exists?(cf, col)
|
158
|
+
klass = OpenStruct.new(:column_family => 'system.schema_columns', :default_consistency => 'QUORUM')
|
159
|
+
cql = DatastaxRails::Cql::ColumnFamily.new(klass)
|
160
|
+
results = CassandraCQL::Result.new(cql.select("index_name").conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf, 'column_name' => col).execute)
|
161
|
+
results.fetch['index_name'] != nil
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
165
|
+
end
|
@@ -1,212 +1,104 @@
|
|
1
1
|
module DatastaxRails
|
2
2
|
module Schema
|
3
|
-
|
4
3
|
class Migrator
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
4
|
+
include DatastaxRails::Schema::Solr
|
5
|
+
include DatastaxRails::Schema::Cassandra
|
6
|
+
|
7
|
+
cattr_accessor :verbose
|
8
|
+
self.verbose = true
|
9
|
+
attr_accessor :errors
|
10
|
+
|
11
|
+
def initialize(keyspace)
|
12
|
+
@keyspace = keyspace
|
13
|
+
check_schema_migrations unless keyspace == 'system'
|
14
|
+
@errors = []
|
15
|
+
end
|
16
|
+
|
17
|
+
def migrate_all(force = false)
|
18
|
+
say_with_time("Migrating all models") do
|
19
|
+
# Ensure all models are loaded (necessary for non-production mode)
|
20
|
+
Dir[Rails.root.join("app","models",'*.rb').to_s].each do |file|
|
21
|
+
require File.basename(file, File.extname(file))
|
22
|
+
end
|
23
|
+
count = 0
|
24
|
+
DatastaxRails::Base.models.each do |m|
|
25
|
+
if !m.abstract_class?
|
26
|
+
count += migrate_one(m, force)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
count
|
15
30
|
end
|
16
31
|
end
|
17
|
-
|
18
|
-
def
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
end
|
45
|
-
|
46
|
-
def self.column_family_tasks
|
47
|
-
cas = DatastaxRails::Base.connection
|
48
|
-
Tasks::ColumnFamily.new(cas.keyspace)
|
49
|
-
end
|
50
|
-
|
51
|
-
def self.get_all_versions
|
52
|
-
cas = DatastaxRails::Base.connection
|
53
|
-
cas.get(schema_migrations_column_family, 'all').map {|(name, _value)| name.to_i}.sort
|
54
|
-
end
|
55
|
-
|
56
|
-
def self.current_version
|
57
|
-
sm_cf = schema_migrations_column_family
|
58
|
-
if column_family_tasks.exists?(sm_cf)
|
59
|
-
get_all_versions.max || 0
|
60
|
-
else
|
61
|
-
0
|
32
|
+
|
33
|
+
def migrate_one(model, force = false)
|
34
|
+
count = 0
|
35
|
+
say_with_time("Migrating #{model.name} to latest version") do
|
36
|
+
if model.payload_model?
|
37
|
+
unless column_family_exists?(model.column_family.to_s)
|
38
|
+
create_payload_column_family(model)
|
39
|
+
count += 1
|
40
|
+
end
|
41
|
+
elsif model.wide_storage_model?
|
42
|
+
unless column_family_exists?(model.column_family.to_s)
|
43
|
+
create_wide_storage_column_family(model)
|
44
|
+
count += 1
|
45
|
+
end
|
46
|
+
count += check_missing_schema(model)
|
47
|
+
elsif model <= DatastaxRails::CassandraOnlyModel
|
48
|
+
unless column_family_exists?(model.column_family.to_s)
|
49
|
+
create_cassandra_column_family(model)
|
50
|
+
count += 1
|
51
|
+
end
|
52
|
+
count += check_key_name(model)
|
53
|
+
count += check_missing_schema(model)
|
54
|
+
else
|
55
|
+
count += check_key_name(model)
|
56
|
+
count += upload_solr_configuration(model, force)
|
57
|
+
count += check_missing_schema(model)
|
58
|
+
end
|
62
59
|
end
|
60
|
+
count
|
63
61
|
end
|
64
62
|
|
65
|
-
|
66
|
-
|
67
|
-
def self.move(direction, migrations_path, steps)
|
68
|
-
migrator = self.new(direction, migrations_path)
|
69
|
-
start_index = migrator.migrations.index(migrator.current_migration)
|
70
|
-
|
71
|
-
if start_index
|
72
|
-
finish = migrator.migrations[start_index + steps]
|
73
|
-
version = finish ? finish.version : 0
|
74
|
-
send(direction, migrations_path, version)
|
75
|
-
end
|
63
|
+
def connection
|
64
|
+
DatastaxRails::Base.connection
|
76
65
|
end
|
77
66
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
cf.comparator_type = 'LongType'
|
67
|
+
private
|
68
|
+
|
69
|
+
# Checks to ensure that the schema_migrations column family exists and creates it if not
|
70
|
+
def check_schema_migrations
|
71
|
+
unless column_family_exists?('schema_migrations')
|
72
|
+
say "Creating schema_migrations column family"
|
73
|
+
connection.execute_cql_query(DatastaxRails::Cql::CreateColumnFamily.new('schema_migrations').key_type(:text).columns(:digest => :text, :solrconfig => :text, :stopwords => :text).to_cql)
|
86
74
|
end
|
75
|
+
|
76
|
+
check_key_name('schema_migrations')
|
87
77
|
end
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
def current_version
|
93
|
-
migrated.last || 0
|
94
|
-
end
|
95
|
-
|
96
|
-
def current_migration
|
97
|
-
migrations.detect { |m| m.version == current_version }
|
98
|
-
end
|
99
|
-
|
100
|
-
def run
|
101
|
-
target = migrations.detect { |m| m.version == @target_version }
|
102
|
-
raise UnknownMigrationVersionError.new(@target_version) if target.nil?
|
103
|
-
unless (up? && migrated.include?(target.version.to_i)) || (down? && !migrated.include?(target.version.to_i))
|
104
|
-
target.migrate(@direction)
|
105
|
-
record_version_state_after_migrating(target)
|
78
|
+
|
79
|
+
def write(text="")
|
80
|
+
puts(text) if verbose
|
106
81
|
end
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
current = migrations.detect { |m| m.version == current_version }
|
111
|
-
target = migrations.detect { |m| m.version == @target_version }
|
112
|
-
|
113
|
-
if target.nil? && !@target_version.nil? && @target_version > 0
|
114
|
-
raise UnknownMigrationVersionError.new(@target_version)
|
82
|
+
|
83
|
+
def say(message, subitem=false)
|
84
|
+
write "#{subitem ? " ->" : "--"} #{message}"
|
115
85
|
end
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
runnable.each do |migration|
|
125
|
-
#puts "Migrating to #{migration.name} (#{migration.version})"
|
126
|
-
|
127
|
-
# On our way up, we skip migrating the ones we've already migrated
|
128
|
-
next if up? && migrated.include?(migration.version.to_i)
|
129
|
-
|
130
|
-
# On our way down, we skip reverting the ones we've never migrated
|
131
|
-
if down? && !migrated.include?(migration.version.to_i)
|
132
|
-
migration.announce 'never migrated, skipping'; migration.write
|
133
|
-
next
|
134
|
-
end
|
135
|
-
|
136
|
-
migration.migrate(@direction)
|
137
|
-
record_version_state_after_migrating(migration)
|
86
|
+
|
87
|
+
def say_with_time(message)
|
88
|
+
say(message)
|
89
|
+
result = nil
|
90
|
+
time = Benchmark.measure { result = yield }
|
91
|
+
say "%.4fs" % time.real, :subitem
|
92
|
+
say("#{result} changes", :subitem) if result.is_a?(Integer)
|
93
|
+
result
|
138
94
|
end
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
migrations = files.inject([]) do |klasses, file|
|
146
|
-
version, name = file.scan(/([0-9]+)_([_a-z0-9]*).rb/).first
|
147
|
-
|
148
|
-
raise IllegalMigrationNameError.new(file) unless version
|
149
|
-
version = version.to_i
|
150
|
-
|
151
|
-
if klasses.detect { |m| m.version == version }
|
152
|
-
raise DuplicateMigrationVersionError.new(version)
|
153
|
-
end
|
154
|
-
|
155
|
-
if klasses.detect { |m| m.name == name.camelize }
|
156
|
-
raise DuplicateMigrationNameError.new(name.camelize)
|
157
|
-
end
|
158
|
-
|
159
|
-
migration = MigrationProxy.new
|
160
|
-
migration.name = name.camelize
|
161
|
-
migration.version = version
|
162
|
-
migration.filename = file
|
163
|
-
klasses << migration
|
164
|
-
end
|
165
|
-
|
166
|
-
migrations = migrations.sort_by { |m| m.version }
|
167
|
-
down? ? migrations.reverse : migrations
|
168
|
-
end
|
169
|
-
end
|
170
|
-
|
171
|
-
def pending_migrations
|
172
|
-
already_migrated = migrated
|
173
|
-
migrations.reject { |m| already_migrated.include?(m.version.to_i) }
|
174
|
-
end
|
175
|
-
|
176
|
-
def migrated
|
177
|
-
@migrated_versions ||= self.class.get_all_versions
|
178
|
-
end
|
179
|
-
|
180
|
-
private
|
181
|
-
|
182
|
-
def column_family_tasks
|
183
|
-
Tasks::ColumnFamily.new(connection.keyspace)
|
184
|
-
end
|
185
|
-
|
186
|
-
def connection
|
187
|
-
DatastaxRails::Base.connection
|
188
|
-
end
|
189
|
-
|
190
|
-
def record_version_state_after_migrating(migration)
|
191
|
-
sm_cf = self.class.schema_migrations_column_family
|
192
|
-
|
193
|
-
@migrated_versions ||= []
|
194
|
-
if down?
|
195
|
-
@migrated_versions.delete(migration.version)
|
196
|
-
connection.remove sm_cf, 'all', migration.version
|
197
|
-
else
|
198
|
-
@migrated_versions.push(migration.version).sort!
|
199
|
-
connection.insert sm_cf, 'all', { migration.version => migration.name }
|
95
|
+
|
96
|
+
def suppress_messages
|
97
|
+
save, self.verbose = verbose, false
|
98
|
+
yield
|
99
|
+
ensure
|
100
|
+
self.verbose = save
|
200
101
|
end
|
201
|
-
end
|
202
|
-
|
203
|
-
def up?
|
204
|
-
@direction == :up
|
205
|
-
end
|
206
|
-
|
207
|
-
def down?
|
208
|
-
@direction == :down
|
209
|
-
end
|
210
102
|
end
|
211
103
|
end
|
212
104
|
end
|
@@ -0,0 +1,158 @@
|
|
1
|
+
module DatastaxRails
|
2
|
+
module Schema
|
3
|
+
module Solr
|
4
|
+
# Generates a SOLR schema file. The default schema template included with DSR can handle
|
5
|
+
# most normal circumstances for indexing. When a customized template is required, it can
|
6
|
+
# be placed in the application's config/solr directory using the naming convention
|
7
|
+
# column_family-schema.xml.erb. It will be processed as a normal ERB file. See the DSR version
|
8
|
+
# for examples.
|
9
|
+
def generate_solr_schema(model)
|
10
|
+
@fields = []
|
11
|
+
@copy_fields = []
|
12
|
+
@fulltext_fields = []
|
13
|
+
@custom_fields = ""
|
14
|
+
model.attribute_definitions.values.each do |attr|
|
15
|
+
coder = attr.coder
|
16
|
+
if coder.options[:solr_type]
|
17
|
+
@fields.push({ :name => attr.name,
|
18
|
+
:type => coder.options[:solr_type].to_s,
|
19
|
+
:indexed => (coder.options[:indexed] == :solr || coder.options[:indexed] == :both).to_s,
|
20
|
+
:stored => coder.options[:stored].to_s,
|
21
|
+
:multi_valued => coder.options[:multi_valued].to_s })
|
22
|
+
end
|
23
|
+
if coder.options[:sortable] && coder.options[:tokenized]
|
24
|
+
@fields.push({ :name => "sort_" + attr.name,
|
25
|
+
:type => "string",
|
26
|
+
:indexed => true,
|
27
|
+
:stored => false,
|
28
|
+
:multi_valued => false })
|
29
|
+
@copy_fields.push({ :source => attr.name, :dest => "sort_" + attr.name }) if (coder.options[:indexed] || coder.options[:stored])
|
30
|
+
end
|
31
|
+
if coder.options[:fulltext]
|
32
|
+
@fulltext_fields << attr.name if (coder.options[:indexed] || coder.options[:stored])
|
33
|
+
end
|
34
|
+
end
|
35
|
+
# Sort the fields so that no matter what order the attributes are arranged into the
|
36
|
+
# same schema file gets generated
|
37
|
+
@fields.sort! {|a,b| a[:name] <=> b[:name]}
|
38
|
+
@copy_fields.sort! {|a,b| a[:source] <=> b[:source]}
|
39
|
+
@fulltext_fields.sort!
|
40
|
+
|
41
|
+
if Rails.root.join('config','solr',"#{model.column_family}-schema.xml.erb").exist?
|
42
|
+
say "Using custom schema for #{model.name}", :subitem
|
43
|
+
ERB.new(Rails.root.join('config','solr',"#{model.column_family}-schema.xml.erb").read).result(binding)
|
44
|
+
else
|
45
|
+
ERB.new(File.read(File.join(File.dirname(__FILE__),"..","..","..","config","schema.xml.erb"))).result(binding)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
# Sends a command to Solr instructing it to reindex the data. The data is reindexed in the background,
|
50
|
+
# and the new index is swapped in once it is finished.
|
51
|
+
def reindex_solr(model)
|
52
|
+
url = "#{DatastaxRails::Base.solr_base_url}/admin/cores?action=RELOAD&name=#{DatastaxRails::Base.config[:keyspace]}.#{model.column_family}&reindex=true&deleteAll=false"
|
53
|
+
say "Posting reindex command to '#{url}'", :subitem
|
54
|
+
`curl -s -X POST '#{url}'`
|
55
|
+
say "Reindexing will run in the background", :subitem
|
56
|
+
end
|
57
|
+
|
58
|
+
# Creates the initial Solr Core. This is required once the first time a Solr schema is uploaded.
|
59
|
+
# It will cause the data to be indexed in the background.
|
60
|
+
def create_solr_core(model)
|
61
|
+
url = "#{DatastaxRails::Base.solr_base_url}/admin/cores?action=CREATE&name=#{DatastaxRails::Base.config[:keyspace]}.#{model.column_family}"
|
62
|
+
say "Posting create command to '#{url}'", :subitem
|
63
|
+
`curl -s -X POST '#{url}'`
|
64
|
+
end
|
65
|
+
|
66
|
+
# Uploads the necessary configuration files for solr to function
|
67
|
+
# The solrconfig and stopwords files can be overridden on a per-model basis
|
68
|
+
# by creating a file called config/solr/column_family-solrconfig.xml or
|
69
|
+
# config/solr/column_family-stopwords.txt
|
70
|
+
#
|
71
|
+
# TODO: find a way to upload arbitrary files automatically (e.g., additional stopwords lists)
|
72
|
+
def upload_solr_configuration(model, force=false)
|
73
|
+
count = 0
|
74
|
+
if Rails.root.join('config','solr',"#{model.column_family}-solrconfig.xml").exist?
|
75
|
+
say "Using custom solrconfig file", :subitem
|
76
|
+
solrconfig = Rails.root.join('config','solr',"#{model.column_family}-solrconfig.xml").read
|
77
|
+
else
|
78
|
+
@legacy = model.legacy_mapping?
|
79
|
+
solrconfig = ERB.new(File.read(File.join(File.dirname(__FILE__),"..","..","..","config","solrconfig.xml.erb"))).result(binding)
|
80
|
+
end
|
81
|
+
if Rails.root.join('config','solr',"#{model.column_family}-stopwords.txt").exist?
|
82
|
+
say "Using custom stopwords file", :subitem
|
83
|
+
stopwords = Rails.root.join('config','solr',"#{model.column_family}-stopwords.txt").read
|
84
|
+
else
|
85
|
+
stopwords = File.read(File.join(File.dirname(__FILE__),"..","..","..","config","stopwords.txt"))
|
86
|
+
end
|
87
|
+
schema = generate_solr_schema(model)
|
88
|
+
solrconfig_digest = Digest::SHA1.hexdigest(solrconfig)
|
89
|
+
stopwords_digest = Digest::SHA1.hexdigest(stopwords)
|
90
|
+
schema_digest = Digest::SHA1.hexdigest(schema)
|
91
|
+
|
92
|
+
newcf = !column_family_exists?(model.column_family.to_s)
|
93
|
+
force ||= newcf
|
94
|
+
|
95
|
+
results = DatastaxRails::Cql::Select.new(SchemaMigration, ['*']).conditions(:key => model.column_family).execute
|
96
|
+
sm_digests = CassandraCQL::Result.new(results).fetch.try(:to_hash) || {}
|
97
|
+
|
98
|
+
solr_url = "#{DatastaxRails::Base.solr_base_url}/resource/#{@keyspace}.#{model.column_family}"
|
99
|
+
|
100
|
+
uri = URI.parse(solr_url)
|
101
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
102
|
+
if uri.scheme == 'https'
|
103
|
+
http.use_ssl = true
|
104
|
+
http.cert = OpenSSL::X509::Certificate.new(Rails.root.join("config","datastax_rails.crt").read)
|
105
|
+
http.key = OpenSSL::PKey::RSA.new(Rails.root.join("config","datastax_rails.key").read)
|
106
|
+
http.ca_path = Rails.root.join("config","sade_ca.crt").to_s
|
107
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
108
|
+
end
|
109
|
+
http.read_timeout = 300
|
110
|
+
|
111
|
+
if force || solrconfig_digest != sm_digests['solrconfig']
|
112
|
+
count += 1
|
113
|
+
loop do
|
114
|
+
say "Posting Solr Config file to '#{solr_url}/solrconfig.xml'", :subitem
|
115
|
+
http.post(uri.path+"/solrconfig.xml", solrconfig)
|
116
|
+
if Rails.env.production?
|
117
|
+
sleep(5)
|
118
|
+
resp = http.get(uri.path+"/solrconfig.xml")
|
119
|
+
continue unless resp.message == 'OK'
|
120
|
+
end
|
121
|
+
break
|
122
|
+
end
|
123
|
+
DatastaxRails::Cql::Update.new(SchemaMigration, model.column_family).columns(:solrconfig => solrconfig_digest).execute
|
124
|
+
end
|
125
|
+
if force || stopwords_digest != sm_digests['stopwords']
|
126
|
+
count += 1
|
127
|
+
loop do
|
128
|
+
say "Posting Solr Stopwords file to '#{solr_url}/stopwords.txt'", :subitem
|
129
|
+
http.post(uri.path+"/stopwords.txt", stopwords)
|
130
|
+
if Rails.env.production?
|
131
|
+
sleep(5)
|
132
|
+
resp = http.get(uri.path+"/stopwords.txt")
|
133
|
+
continue unless resp.message == 'OK'
|
134
|
+
end
|
135
|
+
break
|
136
|
+
end
|
137
|
+
DatastaxRails::Cql::Update.new(SchemaMigration, model.column_family).columns(:stopwords => stopwords_digest).execute
|
138
|
+
end
|
139
|
+
if force || schema_digest != sm_digests['digest']
|
140
|
+
count += 1
|
141
|
+
loop do
|
142
|
+
say "Posting Solr Schema file to '#{solr_url}/schema.xml'", :subitem
|
143
|
+
http.post(uri.path+"/schema.xml", schema)
|
144
|
+
if Rails.env.production?
|
145
|
+
sleep(5)
|
146
|
+
resp = http.get(uri.path+"/schema.xml")
|
147
|
+
continue unless resp.message == 'OK'
|
148
|
+
end
|
149
|
+
break
|
150
|
+
end
|
151
|
+
DatastaxRails::Cql::Update.new(SchemaMigration, model.column_family).columns(:digest => schema_digest).execute
|
152
|
+
newcf ? create_solr_core(model) : reindex_solr(model)
|
153
|
+
end
|
154
|
+
count
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
158
|
+
end
|
@@ -2,36 +2,8 @@ module DatastaxRails
|
|
2
2
|
module Schema
|
3
3
|
extend ActiveSupport::Autoload
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
class DuplicateMigrationVersionError < StandardError#:nodoc:
|
9
|
-
def initialize(version)
|
10
|
-
super("Multiple migrations have the version number #{version}")
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
class DuplicateMigrationNameError < StandardError#:nodoc:
|
15
|
-
def initialize(name)
|
16
|
-
super("Multiple migrations have the name #{name}")
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
class UnknownMigrationVersionError < StandardError #:nodoc:
|
21
|
-
def initialize(version)
|
22
|
-
super("No migration with version number #{version}")
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
class IllegalMigrationNameError < StandardError#:nodoc:
|
27
|
-
def initialize(name)
|
28
|
-
super("Illegal name for migration file: #{name}\n\t(only lower case letters, numbers, and '_' allowed)")
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
5
|
+
autoload :Solr
|
6
|
+
autoload :Cassandra
|
32
7
|
autoload :Migrator
|
33
|
-
autoload :Migration
|
34
|
-
autoload :MigrationProxy
|
35
|
-
|
36
8
|
end
|
37
9
|
end
|