datastax_rails 2.0.12 → 2.0.15
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +5 -5
- data/lib/blankslate.rb +8 -11
- data/lib/cql-rb_extensions.rb +5 -3
- data/lib/datastax_rails/associations/association.rb +93 -101
- data/lib/datastax_rails/associations/association_scope.rb +7 -7
- data/lib/datastax_rails/associations/belongs_to_association.rb +46 -48
- data/lib/datastax_rails/associations/builder/association.rb +32 -31
- data/lib/datastax_rails/associations/builder/belongs_to.rb +19 -20
- data/lib/datastax_rails/associations/builder/collection_association.rb +32 -32
- data/lib/datastax_rails/associations/builder/has_and_belongs_to_many.rb +21 -21
- data/lib/datastax_rails/associations/builder/has_many.rb +39 -40
- data/lib/datastax_rails/associations/builder/has_one.rb +30 -31
- data/lib/datastax_rails/associations/builder/singular_association.rb +31 -33
- data/lib/datastax_rails/associations/collection_association.rb +129 -135
- data/lib/datastax_rails/associations/collection_proxy.rb +21 -21
- data/lib/datastax_rails/associations/has_and_belongs_to_many_association.rb +26 -26
- data/lib/datastax_rails/associations/has_many_association.rb +38 -38
- data/lib/datastax_rails/associations/has_one_association.rb +31 -32
- data/lib/datastax_rails/associations/singular_association.rb +31 -30
- data/lib/datastax_rails/associations.rb +27 -24
- data/lib/datastax_rails/attribute_assignment.rb +17 -17
- data/lib/datastax_rails/attribute_methods/definition.rb +4 -4
- data/lib/datastax_rails/attribute_methods/dirty.rb +34 -33
- data/lib/datastax_rails/attribute_methods/primary_key.rb +3 -8
- data/lib/datastax_rails/attribute_methods/read.rb +10 -12
- data/lib/datastax_rails/attribute_methods/typecasting.rb +36 -35
- data/lib/datastax_rails/attribute_methods/write.rb +5 -6
- data/lib/datastax_rails/attribute_methods.rb +52 -56
- data/lib/datastax_rails/base.rb +122 -125
- data/lib/datastax_rails/callbacks.rb +15 -9
- data/lib/datastax_rails/cassandra_only_model.rb +6 -6
- data/lib/datastax_rails/collection.rb +5 -7
- data/lib/datastax_rails/column.rb +130 -118
- data/lib/datastax_rails/connection/statement_cache.rb +3 -3
- data/lib/datastax_rails/connection.rb +42 -33
- data/lib/datastax_rails/cql/alter_column_family.rb +19 -21
- data/lib/datastax_rails/cql/base.rb +8 -11
- data/lib/datastax_rails/cql/column_family.rb +11 -10
- data/lib/datastax_rails/cql/consistency.rb +2 -2
- data/lib/datastax_rails/cql/create_column_family.rb +15 -15
- data/lib/datastax_rails/cql/create_index.rb +5 -5
- data/lib/datastax_rails/cql/create_keyspace.rb +7 -7
- data/lib/datastax_rails/cql/delete.rb +16 -29
- data/lib/datastax_rails/cql/drop_column_family.rb +2 -2
- data/lib/datastax_rails/cql/drop_index.rb +2 -2
- data/lib/datastax_rails/cql/drop_keyspace.rb +2 -2
- data/lib/datastax_rails/cql/insert.rb +10 -16
- data/lib/datastax_rails/cql/select.rb +21 -33
- data/lib/datastax_rails/cql/truncate.rb +2 -2
- data/lib/datastax_rails/cql/update.rb +16 -24
- data/lib/datastax_rails/cql/use_keyspace.rb +2 -2
- data/lib/datastax_rails/cql.rb +2 -2
- data/lib/datastax_rails/dynamic_model.rb +32 -29
- data/lib/datastax_rails/errors.rb +6 -6
- data/lib/datastax_rails/grouped_collection.rb +3 -3
- data/lib/datastax_rails/inheritance.rb +9 -9
- data/lib/datastax_rails/payload_model.rb +24 -20
- data/lib/datastax_rails/persistence.rb +116 -110
- data/lib/datastax_rails/railtie.rb +7 -7
- data/lib/datastax_rails/reflection.rb +61 -59
- data/lib/datastax_rails/relation/batches.rb +12 -13
- data/lib/datastax_rails/relation/facet_methods.rb +44 -33
- data/lib/datastax_rails/relation/finder_methods.rb +95 -91
- data/lib/datastax_rails/relation/modification_methods.rb +5 -5
- data/lib/datastax_rails/relation/search_methods.rb +102 -102
- data/lib/datastax_rails/relation/spawn_methods.rb +25 -24
- data/lib/datastax_rails/relation/stats_methods.rb +9 -8
- data/lib/datastax_rails/relation.rb +165 -170
- data/lib/datastax_rails/rsolr_client_wrapper.rb +3 -3
- data/lib/datastax_rails/schema/cassandra.rb +44 -43
- data/lib/datastax_rails/schema/migrator.rb +52 -52
- data/lib/datastax_rails/schema/solr.rb +55 -47
- data/lib/datastax_rails/schema_cache.rb +1 -3
- data/lib/datastax_rails/scoping/default.rb +2 -3
- data/lib/datastax_rails/scoping/named.rb +3 -5
- data/lib/datastax_rails/scoping.rb +11 -12
- data/lib/datastax_rails/serialization.rb +34 -31
- data/lib/datastax_rails/serializers/xml_serializer.rb +178 -175
- data/lib/datastax_rails/timestamps.rb +4 -4
- data/lib/datastax_rails/types/dirty_collection.rb +57 -57
- data/lib/datastax_rails/types/dynamic_list.rb +1 -1
- data/lib/datastax_rails/types/dynamic_map.rb +5 -7
- data/lib/datastax_rails/types/dynamic_set.rb +2 -2
- data/lib/datastax_rails/util/solr_repair.rb +3 -3
- data/lib/datastax_rails/validations/associated.rb +8 -6
- data/lib/datastax_rails/validations/uniqueness.rb +8 -8
- data/lib/datastax_rails/validations.rb +9 -10
- data/lib/datastax_rails/version.rb +2 -1
- data/lib/datastax_rails/wide_storage_model.rb +6 -6
- data/lib/datastax_rails.rb +13 -9
- data/lib/schema_migration.rb +3 -3
- data/spec/datastax_rails/associations/belongs_to_association_spec.rb +2 -2
- data/spec/datastax_rails/associations/collection_association_spec.rb +14 -14
- data/spec/datastax_rails/associations/has_many_association_spec.rb +20 -20
- data/spec/datastax_rails/associations_spec.rb +11 -11
- data/spec/datastax_rails/attribute_methods_spec.rb +25 -25
- data/spec/datastax_rails/base_spec.rb +24 -24
- data/spec/datastax_rails/callbacks_spec.rb +21 -21
- data/spec/datastax_rails/column_spec.rb +133 -132
- data/spec/datastax_rails/connection/statement_cache_spec.rb +2 -2
- data/spec/datastax_rails/cql/base_spec.rb +4 -4
- data/spec/datastax_rails/cql/delete_spec.rb +19 -0
- data/spec/datastax_rails/cql/select_spec.rb +8 -8
- data/spec/datastax_rails/cql/update_spec.rb +8 -10
- data/spec/datastax_rails/dynamic_model_spec.rb +36 -22
- data/spec/datastax_rails/inheritance_spec.rb +11 -14
- data/spec/datastax_rails/persistence_spec.rb +73 -74
- data/spec/datastax_rails/relation/batches_spec.rb +13 -13
- data/spec/datastax_rails/relation/facet_methods_spec.rb +43 -35
- data/spec/datastax_rails/relation/finder_methods_spec.rb +77 -78
- data/spec/datastax_rails/relation/modification_methods_spec.rb +19 -19
- data/spec/datastax_rails/relation/search_methods_spec.rb +160 -160
- data/spec/datastax_rails/relation/spawn_methods_spec.rb +18 -18
- data/spec/datastax_rails/relation_spec.rb +119 -116
- data/spec/datastax_rails/schema/migrator_spec.rb +30 -30
- data/spec/datastax_rails/schema/solr_spec.rb +15 -15
- data/spec/datastax_rails/scoping/default_spec.rb +9 -9
- data/spec/datastax_rails/types/dynamic_list_spec.rb +12 -12
- data/spec/datastax_rails/types/dynamic_map_spec.rb +10 -10
- data/spec/datastax_rails/types/dynamic_set_spec.rb +22 -10
- data/spec/datastax_rails/validations/uniqueness_spec.rb +25 -25
- data/spec/datastax_rails/wide_storage_model_spec.rb +11 -0
- data/spec/datastax_rails_spec.rb +2 -2
- data/spec/dummy/config/application.rb +2 -3
- data/spec/dummy/config/boot.rb +1 -1
- data/spec/dummy/config/environments/development.rb +3 -3
- data/spec/dummy/config/environments/test.rb +1 -1
- data/spec/dummy/config/initializers/session_store.rb +1 -1
- data/spec/dummy/config/initializers/wrap_parameters.rb +1 -1
- data/spec/factories/audit_logs.rb +6 -0
- data/spec/factories/hobbies.rb +6 -0
- data/spec/factories/people.rb +5 -0
- data/spec/feature/dynamic_fields_spec.rb +4 -4
- data/spec/feature/overloaded_tables_spec.rb +11 -12
- data/spec/spec_helper.rb +17 -14
- data/spec/support/datastax_test_hook.rb +2 -2
- data/spec/support/default_consistency_shared_examples.rb +11 -11
- data/spec/support/models.rb +31 -32
- metadata +40 -6
- data/lib/datastax_rails/attribute_methods/before_type_cast.rb +0 -71
- data/lib/datastax_rails/log_subscriber.rb +0 -0
- data/spec/dummy/ks/migrate/20111117224534_models.rb +0 -20
@@ -3,11 +3,11 @@ module DatastaxRails
|
|
3
3
|
# a new server tried (if one is available)
|
4
4
|
class RSolrClientWrapper < BlankSlate
|
5
5
|
# @param [RSolr::Client] rsolr the initial RSolr client object to wrap
|
6
|
-
def initialize(rsolr,model)
|
6
|
+
def initialize(rsolr, model)
|
7
7
|
@rsolr = rsolr
|
8
8
|
@model = model
|
9
9
|
end
|
10
|
-
|
10
|
+
|
11
11
|
def method_missing(sym, *args, &block)
|
12
12
|
if @rsolr.uri.host != DatastaxRails::Base.current_server
|
13
13
|
@rsolr.uri.host = DatastaxRails::Base.current_server
|
@@ -26,4 +26,4 @@ module DatastaxRails
|
|
26
26
|
end
|
27
27
|
end
|
28
28
|
end
|
29
|
-
end
|
29
|
+
end
|
@@ -1,8 +1,9 @@
|
|
1
|
+
# rubocop:disable Style/LineLength
|
1
2
|
module DatastaxRails
|
2
3
|
module Schema
|
3
4
|
module Cassandra
|
4
5
|
# Check for missing columns or columns needing cassandra indexes
|
5
|
-
def check_missing_schema(model)
|
6
|
+
def check_missing_schema(model) # rubocop:disable MethodLength
|
6
7
|
count = 0
|
7
8
|
model.attribute_definitions.each do |attribute, definition|
|
8
9
|
unless column_exists?(model.column_family.to_s, attribute.to_s)
|
@@ -10,26 +11,26 @@ module DatastaxRails
|
|
10
11
|
say "Adding column '#{attribute}'", :subitem
|
11
12
|
DatastaxRails::Cql::AlterColumnFamily.new(model.column_family).add(attribute => definition.cql_type).execute
|
12
13
|
end
|
13
|
-
if
|
14
|
+
if definition.options[:cql_index] && !definition.options[:solr_index]
|
14
15
|
unless index_exists?(model.column_family.to_s, attribute.to_s)
|
15
16
|
if index_exists?(model.column_family.to_s, attribute.to_s)
|
16
17
|
count += 1
|
17
|
-
say "Dropping solr index on #{attribute
|
18
|
+
say "Dropping solr index on #{attribute}", :subitem
|
18
19
|
DatastaxRails::Cql::DropIndex.new(solr_index_cql_name(model.column_family.to_s, attribute.to_s)).execute
|
19
20
|
end
|
20
21
|
count += 1
|
21
|
-
say "Creating cassandra index on #{attribute
|
22
|
+
say "Creating cassandra index on #{attribute}", :subitem
|
22
23
|
DatastaxRails::Cql::CreateIndex.new(cassandra_index_cql_name(model.column_family.to_s, attribute.to_s)).on(model.column_family.to_s).column(attribute.to_s).execute
|
23
24
|
end
|
24
|
-
elsif
|
25
|
-
unless column_exists?(model.column_family.to_s, "__#{attribute
|
25
|
+
elsif definition.options[:cql_index]
|
26
|
+
unless column_exists?(model.column_family.to_s, "__#{attribute}")
|
26
27
|
# Create and populate the new column
|
27
28
|
count += 1
|
28
29
|
say "Adding column '__#{attribute}'", :subitem
|
29
|
-
DatastaxRails::Cql::AlterColumnFamily.new(model.column_family).add("__#{attribute
|
30
|
+
DatastaxRails::Cql::AlterColumnFamily.new(model.column_family).add("__#{attribute}" => definition.cql_type).execute
|
30
31
|
say "Populating column '__#{attribute}' (this might take a while)", :subitem
|
31
|
-
export = "echo \"copy #{model.column_family
|
32
|
-
import = "echo \"copy #{model.column_family
|
32
|
+
export = "echo \"copy #{model.column_family} (key, #{attribute}) TO 'dsr_export.csv';\" | cqlsh #{model.current_server}"
|
33
|
+
import = "echo \"copy #{model.column_family} (key, __#{attribute}) FROM 'dsr_export.csv';\" | cqlsh #{model.current_server}"
|
33
34
|
if system(export)
|
34
35
|
system(import)
|
35
36
|
else
|
@@ -37,101 +38,101 @@ module DatastaxRails
|
|
37
38
|
end
|
38
39
|
end
|
39
40
|
count += 1
|
40
|
-
say "Creating cassandra index on __#{attribute
|
41
|
-
DatastaxRails::Cql::CreateIndex.new(cassandra_index_cql_name(model.column_family.to_s, "__#{attribute
|
41
|
+
say "Creating cassandra index on __#{attribute}", :subitem
|
42
|
+
DatastaxRails::Cql::CreateIndex.new(cassandra_index_cql_name(model.column_family.to_s, "__#{attribute}")).on(model.column_family.to_s).column("__#{attribute}").execute
|
42
43
|
end
|
43
44
|
end
|
44
45
|
count
|
45
46
|
end
|
46
|
-
|
47
|
+
|
47
48
|
# Creates a CQL3 backed column family
|
48
49
|
def create_cql3_column_family(model)
|
49
|
-
say
|
50
|
+
say 'Creating Column Family via CQL3', :subitem
|
50
51
|
columns = {}
|
51
|
-
model.attribute_definitions.each {|k,col| columns[k] = col.cql_type}
|
52
|
+
model.attribute_definitions.each { |k, col| columns[k] = col.cql_type }
|
52
53
|
pk = model.primary_key.to_s
|
53
|
-
if
|
54
|
-
pk += ", #{model.cluster_by
|
54
|
+
if model.respond_to?(:cluster_by) && model.cluster_by.present?
|
55
|
+
pk += ", #{model.cluster_by}"
|
55
56
|
end
|
56
57
|
cql = DatastaxRails::Cql::CreateColumnFamily.new(model.column_family).primary_key(pk).columns(columns)
|
57
58
|
cql.with(model.create_options) if model.create_options
|
58
59
|
cql.execute
|
59
60
|
end
|
60
|
-
|
61
|
+
|
61
62
|
# Creates the named keyspace
|
62
63
|
def create_keyspace(keyspace, options = {})
|
63
|
-
opts = { :
|
64
|
-
:
|
64
|
+
opts = { name: keyspace.to_s,
|
65
|
+
strategy_class: 'org.apache.cassandra.locator.NetworkTopologyStrategy' }.with_indifferent_access.merge(options)
|
65
66
|
|
66
|
-
if
|
67
|
-
say "Keyspace #{keyspace
|
67
|
+
if keyspace_exists?(keyspace.to_s)
|
68
|
+
say "Keyspace #{keyspace} already exists"
|
68
69
|
return false
|
69
70
|
else
|
70
71
|
cql = DatastaxRails::Cql::CreateKeyspace.new(opts.delete(:name))
|
71
72
|
cql.strategy_class(opts.delete(:strategy_class))
|
72
73
|
strategy_options = opts.delete('strategy_options')
|
73
74
|
cql.strategy_options(strategy_options.symbolize_keys)
|
74
|
-
say "Creating keyspace #{keyspace
|
75
|
+
say "Creating keyspace #{keyspace}"
|
75
76
|
cql.execute
|
76
77
|
return true
|
77
78
|
end
|
78
79
|
end
|
79
|
-
|
80
|
+
|
80
81
|
def drop_keyspace
|
81
|
-
say "Dropping keyspace #{@keyspace
|
82
|
+
say "Dropping keyspace #{@keyspace}"
|
82
83
|
DatastaxRails::Cql::DropKeyspace.new(@keyspace.to_s).execute
|
83
84
|
end
|
84
|
-
|
85
|
+
|
85
86
|
# Computes the expected solr index name as reported by CQL.
|
86
87
|
def solr_index_cql_name(cf, column)
|
87
|
-
"#{@keyspace}_#{cf
|
88
|
+
"#{@keyspace}_#{cf}_#{column}_index"
|
88
89
|
end
|
89
|
-
|
90
|
+
|
90
91
|
# Computes the expected cassandra index name as reported by CQL.
|
91
92
|
def cassandra_index_cql_name(cf, column)
|
92
|
-
"#{cf
|
93
|
+
"#{cf}_#{column}_idx"
|
93
94
|
end
|
94
|
-
|
95
|
+
|
95
96
|
# Checks the Cassandra system tables to see if a keyspace exists
|
96
97
|
def keyspace_exists?(keyspace)
|
97
|
-
klass = OpenStruct.new(:
|
98
|
+
klass = OpenStruct.new(column_family: 'system.schema_keyspaces', default_consistency: 'QUORUM')
|
98
99
|
cql = DatastaxRails::Cql::ColumnFamily.new(klass)
|
99
|
-
results = cql.select(
|
100
|
+
results = cql.select('count(*)').conditions('keyspace_name' => keyspace).execute
|
100
101
|
results.first['count'].to_i > 0
|
101
102
|
end
|
102
|
-
|
103
|
+
|
103
104
|
# Checks the Cassandra system tables to see if a column family exists
|
104
105
|
def column_family_exists?(cf)
|
105
|
-
klass = OpenStruct.new(:
|
106
|
+
klass = OpenStruct.new(column_family: 'system.schema_columnfamilies', default_consistency: 'QUORUM')
|
106
107
|
cql = DatastaxRails::Cql::ColumnFamily.new(klass)
|
107
|
-
results = cql.select(
|
108
|
+
results = cql.select('count(*)').conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf).execute
|
108
109
|
results.first['count'] > 0
|
109
110
|
end
|
110
|
-
|
111
|
+
|
111
112
|
# Checks the Cassandra system tables to see if a column exists on a column family
|
112
113
|
def column_exists?(cf, col)
|
113
|
-
klass = OpenStruct.new(:
|
114
|
+
klass = OpenStruct.new(column_family: 'system.schema_columns', default_consistency: 'QUORUM')
|
114
115
|
cql = DatastaxRails::Cql::ColumnFamily.new(klass)
|
115
|
-
results = cql.select(
|
116
|
+
results = cql.select('count(*)').conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf, 'column_name' => col).execute
|
116
117
|
exists = results.first['count'] > 0
|
117
118
|
unless exists
|
118
119
|
# We need to check if it's part of an alias (ugh)
|
119
|
-
klass = OpenStruct.new(:
|
120
|
+
klass = OpenStruct.new(column_family: 'system.schema_columnfamilies', default_consistency: 'QUORUM')
|
120
121
|
cql = DatastaxRails::Cql::ColumnFamily.new(klass)
|
121
|
-
results = cql.select(
|
122
|
+
results = cql.select('column_aliases, key_aliases, value_alias').conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf).execute
|
122
123
|
row = results.first
|
123
124
|
exists = row['key_aliases'].include?(col.to_s) || row['column_aliases'].include?(col.to_s) || (row['value_alias'] && row['value_alias'].include?(col.to_s))
|
124
125
|
end
|
125
126
|
exists
|
126
127
|
end
|
127
|
-
|
128
|
+
|
128
129
|
# Checks the Cassandra system tables to see if an index exists on a column family
|
129
130
|
def index_exists?(cf, col)
|
130
|
-
klass = OpenStruct.new(:
|
131
|
+
klass = OpenStruct.new(column_family: 'system.schema_columns', default_consistency: 'QUORUM')
|
131
132
|
cql = DatastaxRails::Cql::ColumnFamily.new(klass)
|
132
|
-
results = cql.select(
|
133
|
+
results = cql.select('index_name').conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf, 'column_name' => col).execute
|
133
134
|
results.first['index_name'] != nil
|
134
135
|
end
|
135
136
|
end
|
136
137
|
end
|
137
|
-
end
|
138
|
+
end
|
@@ -1,13 +1,15 @@
|
|
1
1
|
module DatastaxRails
|
2
2
|
module Schema
|
3
|
+
# DatastaxRails reads the attributes from the individual models. This class migrates both Cassandra
|
4
|
+
# and Solr to the point where they reflect what is specified in the models.
|
3
5
|
class Migrator
|
4
6
|
include DatastaxRails::Schema::Solr
|
5
7
|
include DatastaxRails::Schema::Cassandra
|
6
|
-
|
8
|
+
|
7
9
|
cattr_accessor :verbose
|
8
10
|
self.verbose = true
|
9
11
|
attr_accessor :errors
|
10
|
-
|
12
|
+
|
11
13
|
def initialize(keyspace)
|
12
14
|
@keyspace = keyspace
|
13
15
|
check_schema_migrations unless keyspace == 'system'
|
@@ -15,7 +17,7 @@ module DatastaxRails
|
|
15
17
|
end
|
16
18
|
|
17
19
|
def migrate_all(force = false)
|
18
|
-
say_with_time(
|
20
|
+
say_with_time('Migrating all models') do
|
19
21
|
|
20
22
|
FileList[rails_models].each do |model|
|
21
23
|
require model
|
@@ -23,14 +25,12 @@ module DatastaxRails
|
|
23
25
|
|
24
26
|
count = 0
|
25
27
|
DatastaxRails::Base.models.each do |m|
|
26
|
-
|
27
|
-
count += migrate_one(m, force)
|
28
|
-
end
|
28
|
+
count += migrate_one(m, force) unless m.abstract_class?
|
29
29
|
end
|
30
30
|
count
|
31
31
|
end
|
32
32
|
end
|
33
|
-
|
33
|
+
|
34
34
|
def migrate_one(model, force = false)
|
35
35
|
count = 0
|
36
36
|
say_with_time("Migrating #{model.name} to latest version") do
|
@@ -38,9 +38,9 @@ module DatastaxRails
|
|
38
38
|
create_cql3_column_family(model)
|
39
39
|
count += 1
|
40
40
|
end
|
41
|
-
|
41
|
+
|
42
42
|
count += check_missing_schema(model)
|
43
|
-
|
43
|
+
|
44
44
|
unless model <= DatastaxRails::CassandraOnlyModel
|
45
45
|
count += upload_solr_configuration(model, force)
|
46
46
|
end
|
@@ -54,49 +54,49 @@ module DatastaxRails
|
|
54
54
|
|
55
55
|
private
|
56
56
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
57
|
+
# Determine all models to be included within the migration
|
58
|
+
# using Rails config paths instead of absolute paths.
|
59
|
+
# This enables Rails Engines to monkey patch their own
|
60
|
+
# models in, to be automatically included within migrations.
|
61
|
+
#
|
62
|
+
# @see http://pivotallabs.com/leave-your-migrations-in-your-rails-engines/
|
63
|
+
#
|
64
|
+
# @return [Array] list of configured application models
|
65
|
+
def rails_models
|
66
|
+
Rails.configuration.paths['app/models'].expanded.map { |p| p + '/*.rb' }
|
67
|
+
end
|
68
|
+
|
69
|
+
# Checks to ensure that the schema_migrations column family exists and creates it if not
|
70
|
+
def check_schema_migrations
|
71
|
+
return if column_family_exists?('schema_migrations')
|
72
|
+
say 'Creating schema_migrations column family'
|
73
|
+
DatastaxRails::Cql::CreateColumnFamily.new('schema_migrations').primary_key('cf')
|
74
|
+
.columns(cf: :text, digest: :text, solrconfig: :text, stopwords: :text).execute
|
75
|
+
end
|
76
|
+
|
77
|
+
def write(text = '')
|
78
|
+
puts(text) if verbose
|
79
|
+
end
|
80
|
+
|
81
|
+
def say(message, subitem = false)
|
82
|
+
write "#{subitem ? ' ->' : '--'} #{message}"
|
83
|
+
end
|
84
|
+
|
85
|
+
def say_with_time(message)
|
86
|
+
say(message)
|
87
|
+
result = nil
|
88
|
+
time = Benchmark.measure { result = yield }
|
89
|
+
say format('%.4fs', time.real), :subitem
|
90
|
+
say("#{result} changes", :subitem) if result.is_a?(Integer)
|
91
|
+
result
|
92
|
+
end
|
93
|
+
|
94
|
+
def suppress_messages
|
95
|
+
save, self.verbose = verbose, false
|
96
|
+
yield
|
97
|
+
ensure
|
98
|
+
self.verbose = save
|
99
|
+
end
|
100
100
|
end
|
101
101
|
end
|
102
102
|
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# rubocop:disable Style/LineLength
|
1
2
|
module DatastaxRails
|
2
3
|
module Schema
|
3
4
|
module Solr
|
@@ -5,7 +6,7 @@ module DatastaxRails
|
|
5
6
|
# most normal circumstances for indexing. When a customized template is required, it can
|
6
7
|
# be placed in the application's config/solr directory using the naming convention
|
7
8
|
# column_family-schema.xml.erb. It will be processed as a normal ERB file. See the DSR version
|
8
|
-
# for examples.
|
9
|
+
# for examples.
|
9
10
|
def generate_solr_schema(model)
|
10
11
|
@fields = []
|
11
12
|
@copy_fields = []
|
@@ -15,32 +16,34 @@ module DatastaxRails
|
|
15
16
|
else
|
16
17
|
@primary_key = model.primary_key
|
17
18
|
end
|
18
|
-
@custom_fields =
|
19
|
+
@custom_fields = ''
|
19
20
|
@columns = model.attribute_definitions.values
|
20
|
-
@fields.sort! {|a,b| a[:name] <=> b[:name]}
|
21
|
-
@copy_fields.sort! {|a,b| a[:source] <=> b[:source]}
|
21
|
+
@fields.sort! { |a, b| a[:name] <=> b[:name] }
|
22
|
+
@copy_fields.sort! { |a, b| a[:source] <=> b[:source] }
|
22
23
|
@fulltext_fields.sort!
|
23
|
-
|
24
|
-
if Rails.root.join('config','solr',"#{model.column_family}-schema.xml.erb").exist?
|
24
|
+
|
25
|
+
if Rails.root.join('config', 'solr', "#{model.column_family}-schema.xml.erb").exist?
|
25
26
|
say "Using custom schema for #{model.name}", :subitem
|
26
|
-
ERB.new(Rails.root.join('config','solr',"#{model.column_family}-schema.xml.erb").read, 0, '>')
|
27
|
-
|
27
|
+
ERB.new(Rails.root.join('config', 'solr', "#{model.column_family}-schema.xml.erb").read, 0, '>')
|
28
|
+
.result(binding)
|
29
|
+
elsif Rails.root.join('config', 'solr', 'application-schema.xml.erb').exist?
|
28
30
|
say 'Using application default schema', :subitem
|
29
|
-
ERB.new(Rails.root.join('config','solr','application-schema.xml.erb').read, 0, '>').result(binding)
|
31
|
+
ERB.new(Rails.root.join('config', 'solr', 'application-schema.xml.erb').read, 0, '>').result(binding)
|
30
32
|
else
|
31
|
-
ERB.new(File.read(File.join(File.dirname(__FILE__),
|
33
|
+
ERB.new(File.read(File.join(File.dirname(__FILE__), '..', '..', '..', 'config', 'schema.xml.erb')), 0, '>')
|
34
|
+
.result(binding)
|
32
35
|
end
|
33
36
|
end
|
34
|
-
|
37
|
+
|
35
38
|
# Sends a command to Solr instructing it to reindex the data. The data is reindexed in the background,
|
36
39
|
# and the new index is swapped in once it is finished.
|
37
|
-
def reindex_solr(model, destructive=false)
|
38
|
-
url = "#{DatastaxRails::Base.solr_base_url}/admin/cores?action=RELOAD&name=#{DatastaxRails::Base.config[:keyspace]}.#{model.column_family}&reindex=true&deleteAll=#{destructive
|
40
|
+
def reindex_solr(model, destructive = false)
|
41
|
+
url = "#{DatastaxRails::Base.solr_base_url}/admin/cores?action=RELOAD&name=#{DatastaxRails::Base.config[:keyspace]}.#{model.column_family}&reindex=true&deleteAll=#{destructive}"
|
39
42
|
say "Posting reindex command to '#{url}'", :subitem
|
40
43
|
`curl -s -X POST '#{url}'`
|
41
|
-
say
|
44
|
+
say 'Reindexing will run in the background', :subitem
|
42
45
|
end
|
43
|
-
|
46
|
+
|
44
47
|
# Creates the initial Solr Core. This is required once the first time a Solr schema is uploaded.
|
45
48
|
# It will cause the data to be indexed in the background.
|
46
49
|
def create_solr_core(model)
|
@@ -48,97 +51,102 @@ module DatastaxRails
|
|
48
51
|
say "Posting create command to '#{url}'", :subitem
|
49
52
|
`curl -s -X POST '#{url}'`
|
50
53
|
end
|
51
|
-
|
54
|
+
|
52
55
|
# Uploads the necessary configuration files for solr to function
|
53
56
|
# The solrconfig and stopwords files can be overridden on a per-model basis
|
54
57
|
# by creating a file called config/solr/column_family-solrconfig.xml or
|
55
58
|
# config/solr/column_family-stopwords.txt
|
56
59
|
#
|
57
60
|
# TODO: find a way to upload arbitrary files automatically (e.g., additional stopwords lists)
|
58
|
-
|
61
|
+
# TODO: Simplify this method
|
62
|
+
def upload_solr_configuration(model, force = false, reindex = true) # rubocop:disable all
|
59
63
|
count = 0
|
60
|
-
if Rails.root.join('config','solr',"#{model.column_family}-solrconfig.xml").exist?
|
61
|
-
say
|
62
|
-
solrconfig = Rails.root.join('config','solr',"#{model.column_family}-solrconfig.xml").read
|
64
|
+
if Rails.root.join('config', 'solr', "#{model.column_family}-solrconfig.xml").exist?
|
65
|
+
say 'Using custom solrconfig file', :subitem
|
66
|
+
solrconfig = Rails.root.join('config', 'solr', "#{model.column_family}-solrconfig.xml").read
|
63
67
|
else
|
64
68
|
@legacy = model.legacy_mapping?
|
65
|
-
solrconfig = ERB.new(File.read(File.join(File.dirname(__FILE__),
|
69
|
+
solrconfig = ERB.new(File.read(File.join(File.dirname(__FILE__), '..', '..', '..', 'config', 'solrconfig.xml.erb'))).result(binding)
|
66
70
|
end
|
67
|
-
if Rails.root.join('config','solr',"#{model.column_family}-stopwords.txt").exist?
|
68
|
-
say
|
69
|
-
stopwords = Rails.root.join('config','solr',"#{model.column_family}-stopwords.txt").read
|
71
|
+
if Rails.root.join('config', 'solr', "#{model.column_family}-stopwords.txt").exist?
|
72
|
+
say 'Using custom stopwords file', :subitem
|
73
|
+
stopwords = Rails.root.join('config', 'solr', "#{model.column_family}-stopwords.txt").read
|
70
74
|
else
|
71
|
-
stopwords = File.read(File.join(File.dirname(__FILE__),
|
75
|
+
stopwords = File.read(File.join(File.dirname(__FILE__), '..', '..', '..', 'config', 'stopwords.txt'))
|
72
76
|
end
|
73
77
|
schema = generate_solr_schema(model)
|
74
78
|
solrconfig_digest = Digest::SHA1.hexdigest(solrconfig)
|
75
79
|
stopwords_digest = Digest::SHA1.hexdigest(stopwords)
|
76
80
|
schema_digest = Digest::SHA1.hexdigest(schema)
|
77
|
-
|
81
|
+
|
78
82
|
newcf = !column_exists?(model.column_family, 'solr_query')
|
79
83
|
force ||= newcf
|
80
|
-
|
81
|
-
results = DatastaxRails::Cql::Select.new(SchemaMigration, ['*']).conditions(:
|
84
|
+
|
85
|
+
results = DatastaxRails::Cql::Select.new(SchemaMigration, ['*']).conditions(cf: model.column_family).execute
|
82
86
|
sm_digests = results.first || {}
|
83
|
-
|
87
|
+
|
84
88
|
solr_url = "#{DatastaxRails::Base.solr_base_url}/resource/#{@keyspace}.#{model.column_family}"
|
85
|
-
|
89
|
+
|
86
90
|
uri = URI.parse(solr_url)
|
87
91
|
http = Net::HTTP.new(uri.host, uri.port)
|
88
92
|
if uri.scheme == 'https'
|
89
93
|
http.use_ssl = true
|
90
|
-
http.cert = OpenSSL::X509::Certificate.new(Rails.root.join(
|
91
|
-
http.key = OpenSSL::PKey::RSA.new(Rails.root.join(
|
92
|
-
http.ca_path = Rails.root.join(
|
94
|
+
http.cert = OpenSSL::X509::Certificate.new(Rails.root.join('config', 'datastax_rails.crt').read)
|
95
|
+
http.key = OpenSSL::PKey::RSA.new(Rails.root.join('config', 'datastax_rails.key').read)
|
96
|
+
http.ca_path = Rails.root.join('config', 'sade_ca.crt').to_s
|
93
97
|
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
94
98
|
end
|
95
99
|
http.read_timeout = 300
|
96
|
-
|
100
|
+
|
97
101
|
if force || solrconfig_digest != sm_digests['solrconfig']
|
98
102
|
count += 1
|
99
|
-
loop do
|
103
|
+
loop do
|
100
104
|
say "Posting Solr Config file to '#{solr_url}/solrconfig.xml'", :subitem
|
101
|
-
http.post(uri.path+
|
105
|
+
http.post(uri.path + '/solrconfig.xml', solrconfig)
|
102
106
|
if Rails.env.production?
|
103
107
|
sleep(5)
|
104
|
-
resp = http.get(uri.path+
|
105
|
-
continue unless resp.message == 'OK'
|
108
|
+
resp = http.get(uri.path + '/solrconfig.xml')
|
109
|
+
continue unless resp.message == 'OK'
|
106
110
|
end
|
107
111
|
break
|
108
112
|
end
|
109
|
-
DatastaxRails::Cql::Update.new(SchemaMigration, :
|
113
|
+
DatastaxRails::Cql::Update.new(SchemaMigration, cf: model.column_family).columns(solrconfig: solrconfig_digest).execute
|
110
114
|
end
|
111
115
|
if force || stopwords_digest != sm_digests['stopwords']
|
112
116
|
count += 1
|
113
117
|
loop do
|
114
118
|
say "Posting Solr Stopwords file to '#{solr_url}/stopwords.txt'", :subitem
|
115
|
-
http.post(uri.path+
|
119
|
+
http.post(uri.path + '/stopwords.txt', stopwords)
|
116
120
|
if Rails.env.production?
|
117
121
|
sleep(5)
|
118
|
-
resp = http.get(uri.path+
|
122
|
+
resp = http.get(uri.path + '/stopwords.txt')
|
119
123
|
continue unless resp.message == 'OK'
|
120
124
|
end
|
121
125
|
break
|
122
126
|
end
|
123
|
-
DatastaxRails::Cql::Update.new(SchemaMigration, :
|
127
|
+
DatastaxRails::Cql::Update.new(SchemaMigration, cf: model.column_family).columns(stopwords: stopwords_digest).execute
|
124
128
|
end
|
125
129
|
if force || schema_digest != sm_digests['digest']
|
126
130
|
count += 1
|
127
131
|
loop do
|
128
132
|
say "Posting Solr Schema file to '#{solr_url}/schema.xml'", :subitem
|
129
|
-
http.post(uri.path+
|
133
|
+
http.post(uri.path + '/schema.xml', schema)
|
130
134
|
if Rails.env.production?
|
131
135
|
sleep(5)
|
132
|
-
resp = http.get(uri.path+
|
136
|
+
resp = http.get(uri.path + '/schema.xml')
|
133
137
|
continue unless resp.message == 'OK'
|
134
138
|
end
|
135
139
|
break
|
136
140
|
end
|
137
|
-
DatastaxRails::Cql::Update.new(SchemaMigration, :
|
138
|
-
newcf
|
141
|
+
DatastaxRails::Cql::Update.new(SchemaMigration, cf: model.column_family).columns(digest: schema_digest).execute
|
142
|
+
if newcf
|
143
|
+
create_solr_core(model)
|
144
|
+
elsif reindex
|
145
|
+
reindex_solr(model)
|
146
|
+
end
|
139
147
|
end
|
140
148
|
count
|
141
149
|
end
|
142
150
|
end
|
143
151
|
end
|
144
|
-
end
|
152
|
+
end
|