datastax_rails 2.0.12 → 2.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +5 -5
- data/lib/blankslate.rb +8 -11
- data/lib/cql-rb_extensions.rb +5 -3
- data/lib/datastax_rails/associations/association.rb +93 -101
- data/lib/datastax_rails/associations/association_scope.rb +7 -7
- data/lib/datastax_rails/associations/belongs_to_association.rb +46 -48
- data/lib/datastax_rails/associations/builder/association.rb +32 -31
- data/lib/datastax_rails/associations/builder/belongs_to.rb +19 -20
- data/lib/datastax_rails/associations/builder/collection_association.rb +32 -32
- data/lib/datastax_rails/associations/builder/has_and_belongs_to_many.rb +21 -21
- data/lib/datastax_rails/associations/builder/has_many.rb +39 -40
- data/lib/datastax_rails/associations/builder/has_one.rb +30 -31
- data/lib/datastax_rails/associations/builder/singular_association.rb +31 -33
- data/lib/datastax_rails/associations/collection_association.rb +129 -135
- data/lib/datastax_rails/associations/collection_proxy.rb +21 -21
- data/lib/datastax_rails/associations/has_and_belongs_to_many_association.rb +26 -26
- data/lib/datastax_rails/associations/has_many_association.rb +38 -38
- data/lib/datastax_rails/associations/has_one_association.rb +31 -32
- data/lib/datastax_rails/associations/singular_association.rb +31 -30
- data/lib/datastax_rails/associations.rb +27 -24
- data/lib/datastax_rails/attribute_assignment.rb +17 -17
- data/lib/datastax_rails/attribute_methods/definition.rb +4 -4
- data/lib/datastax_rails/attribute_methods/dirty.rb +34 -33
- data/lib/datastax_rails/attribute_methods/primary_key.rb +3 -8
- data/lib/datastax_rails/attribute_methods/read.rb +10 -12
- data/lib/datastax_rails/attribute_methods/typecasting.rb +36 -35
- data/lib/datastax_rails/attribute_methods/write.rb +5 -6
- data/lib/datastax_rails/attribute_methods.rb +52 -56
- data/lib/datastax_rails/base.rb +122 -125
- data/lib/datastax_rails/callbacks.rb +15 -9
- data/lib/datastax_rails/cassandra_only_model.rb +6 -6
- data/lib/datastax_rails/collection.rb +5 -7
- data/lib/datastax_rails/column.rb +130 -118
- data/lib/datastax_rails/connection/statement_cache.rb +3 -3
- data/lib/datastax_rails/connection.rb +42 -33
- data/lib/datastax_rails/cql/alter_column_family.rb +19 -21
- data/lib/datastax_rails/cql/base.rb +8 -11
- data/lib/datastax_rails/cql/column_family.rb +11 -10
- data/lib/datastax_rails/cql/consistency.rb +2 -2
- data/lib/datastax_rails/cql/create_column_family.rb +15 -15
- data/lib/datastax_rails/cql/create_index.rb +5 -5
- data/lib/datastax_rails/cql/create_keyspace.rb +7 -7
- data/lib/datastax_rails/cql/delete.rb +16 -29
- data/lib/datastax_rails/cql/drop_column_family.rb +2 -2
- data/lib/datastax_rails/cql/drop_index.rb +2 -2
- data/lib/datastax_rails/cql/drop_keyspace.rb +2 -2
- data/lib/datastax_rails/cql/insert.rb +10 -16
- data/lib/datastax_rails/cql/select.rb +21 -33
- data/lib/datastax_rails/cql/truncate.rb +2 -2
- data/lib/datastax_rails/cql/update.rb +16 -24
- data/lib/datastax_rails/cql/use_keyspace.rb +2 -2
- data/lib/datastax_rails/cql.rb +2 -2
- data/lib/datastax_rails/dynamic_model.rb +32 -29
- data/lib/datastax_rails/errors.rb +6 -6
- data/lib/datastax_rails/grouped_collection.rb +3 -3
- data/lib/datastax_rails/inheritance.rb +9 -9
- data/lib/datastax_rails/payload_model.rb +24 -20
- data/lib/datastax_rails/persistence.rb +116 -110
- data/lib/datastax_rails/railtie.rb +7 -7
- data/lib/datastax_rails/reflection.rb +61 -59
- data/lib/datastax_rails/relation/batches.rb +12 -13
- data/lib/datastax_rails/relation/facet_methods.rb +44 -33
- data/lib/datastax_rails/relation/finder_methods.rb +95 -91
- data/lib/datastax_rails/relation/modification_methods.rb +5 -5
- data/lib/datastax_rails/relation/search_methods.rb +102 -102
- data/lib/datastax_rails/relation/spawn_methods.rb +25 -24
- data/lib/datastax_rails/relation/stats_methods.rb +9 -8
- data/lib/datastax_rails/relation.rb +165 -170
- data/lib/datastax_rails/rsolr_client_wrapper.rb +3 -3
- data/lib/datastax_rails/schema/cassandra.rb +44 -43
- data/lib/datastax_rails/schema/migrator.rb +52 -52
- data/lib/datastax_rails/schema/solr.rb +55 -47
- data/lib/datastax_rails/schema_cache.rb +1 -3
- data/lib/datastax_rails/scoping/default.rb +2 -3
- data/lib/datastax_rails/scoping/named.rb +3 -5
- data/lib/datastax_rails/scoping.rb +11 -12
- data/lib/datastax_rails/serialization.rb +34 -31
- data/lib/datastax_rails/serializers/xml_serializer.rb +178 -175
- data/lib/datastax_rails/timestamps.rb +4 -4
- data/lib/datastax_rails/types/dirty_collection.rb +57 -57
- data/lib/datastax_rails/types/dynamic_list.rb +1 -1
- data/lib/datastax_rails/types/dynamic_map.rb +5 -7
- data/lib/datastax_rails/types/dynamic_set.rb +2 -2
- data/lib/datastax_rails/util/solr_repair.rb +3 -3
- data/lib/datastax_rails/validations/associated.rb +8 -6
- data/lib/datastax_rails/validations/uniqueness.rb +8 -8
- data/lib/datastax_rails/validations.rb +9 -10
- data/lib/datastax_rails/version.rb +2 -1
- data/lib/datastax_rails/wide_storage_model.rb +6 -6
- data/lib/datastax_rails.rb +13 -9
- data/lib/schema_migration.rb +3 -3
- data/spec/datastax_rails/associations/belongs_to_association_spec.rb +2 -2
- data/spec/datastax_rails/associations/collection_association_spec.rb +14 -14
- data/spec/datastax_rails/associations/has_many_association_spec.rb +20 -20
- data/spec/datastax_rails/associations_spec.rb +11 -11
- data/spec/datastax_rails/attribute_methods_spec.rb +25 -25
- data/spec/datastax_rails/base_spec.rb +24 -24
- data/spec/datastax_rails/callbacks_spec.rb +21 -21
- data/spec/datastax_rails/column_spec.rb +133 -132
- data/spec/datastax_rails/connection/statement_cache_spec.rb +2 -2
- data/spec/datastax_rails/cql/base_spec.rb +4 -4
- data/spec/datastax_rails/cql/delete_spec.rb +19 -0
- data/spec/datastax_rails/cql/select_spec.rb +8 -8
- data/spec/datastax_rails/cql/update_spec.rb +8 -10
- data/spec/datastax_rails/dynamic_model_spec.rb +36 -22
- data/spec/datastax_rails/inheritance_spec.rb +11 -14
- data/spec/datastax_rails/persistence_spec.rb +73 -74
- data/spec/datastax_rails/relation/batches_spec.rb +13 -13
- data/spec/datastax_rails/relation/facet_methods_spec.rb +43 -35
- data/spec/datastax_rails/relation/finder_methods_spec.rb +77 -78
- data/spec/datastax_rails/relation/modification_methods_spec.rb +19 -19
- data/spec/datastax_rails/relation/search_methods_spec.rb +160 -160
- data/spec/datastax_rails/relation/spawn_methods_spec.rb +18 -18
- data/spec/datastax_rails/relation_spec.rb +119 -116
- data/spec/datastax_rails/schema/migrator_spec.rb +30 -30
- data/spec/datastax_rails/schema/solr_spec.rb +15 -15
- data/spec/datastax_rails/scoping/default_spec.rb +9 -9
- data/spec/datastax_rails/types/dynamic_list_spec.rb +12 -12
- data/spec/datastax_rails/types/dynamic_map_spec.rb +10 -10
- data/spec/datastax_rails/types/dynamic_set_spec.rb +22 -10
- data/spec/datastax_rails/validations/uniqueness_spec.rb +25 -25
- data/spec/datastax_rails/wide_storage_model_spec.rb +11 -0
- data/spec/datastax_rails_spec.rb +2 -2
- data/spec/dummy/config/application.rb +2 -3
- data/spec/dummy/config/boot.rb +1 -1
- data/spec/dummy/config/environments/development.rb +3 -3
- data/spec/dummy/config/environments/test.rb +1 -1
- data/spec/dummy/config/initializers/session_store.rb +1 -1
- data/spec/dummy/config/initializers/wrap_parameters.rb +1 -1
- data/spec/factories/audit_logs.rb +6 -0
- data/spec/factories/hobbies.rb +6 -0
- data/spec/factories/people.rb +5 -0
- data/spec/feature/dynamic_fields_spec.rb +4 -4
- data/spec/feature/overloaded_tables_spec.rb +11 -12
- data/spec/spec_helper.rb +17 -14
- data/spec/support/datastax_test_hook.rb +2 -2
- data/spec/support/default_consistency_shared_examples.rb +11 -11
- data/spec/support/models.rb +31 -32
- metadata +40 -6
- data/lib/datastax_rails/attribute_methods/before_type_cast.rb +0 -71
- data/lib/datastax_rails/log_subscriber.rb +0 -0
- data/spec/dummy/ks/migrate/20111117224534_models.rb +0 -20
@@ -3,11 +3,11 @@ module DatastaxRails
|
|
3
3
|
# a new server tried (if one is available)
|
4
4
|
class RSolrClientWrapper < BlankSlate
|
5
5
|
# @param [RSolr::Client] rsolr the initial RSolr client object to wrap
|
6
|
-
def initialize(rsolr,model)
|
6
|
+
def initialize(rsolr, model)
|
7
7
|
@rsolr = rsolr
|
8
8
|
@model = model
|
9
9
|
end
|
10
|
-
|
10
|
+
|
11
11
|
def method_missing(sym, *args, &block)
|
12
12
|
if @rsolr.uri.host != DatastaxRails::Base.current_server
|
13
13
|
@rsolr.uri.host = DatastaxRails::Base.current_server
|
@@ -26,4 +26,4 @@ module DatastaxRails
|
|
26
26
|
end
|
27
27
|
end
|
28
28
|
end
|
29
|
-
end
|
29
|
+
end
|
@@ -1,8 +1,9 @@
|
|
1
|
+
# rubocop:disable Style/LineLength
|
1
2
|
module DatastaxRails
|
2
3
|
module Schema
|
3
4
|
module Cassandra
|
4
5
|
# Check for missing columns or columns needing cassandra indexes
|
5
|
-
def check_missing_schema(model)
|
6
|
+
def check_missing_schema(model) # rubocop:disable MethodLength
|
6
7
|
count = 0
|
7
8
|
model.attribute_definitions.each do |attribute, definition|
|
8
9
|
unless column_exists?(model.column_family.to_s, attribute.to_s)
|
@@ -10,26 +11,26 @@ module DatastaxRails
|
|
10
11
|
say "Adding column '#{attribute}'", :subitem
|
11
12
|
DatastaxRails::Cql::AlterColumnFamily.new(model.column_family).add(attribute => definition.cql_type).execute
|
12
13
|
end
|
13
|
-
if
|
14
|
+
if definition.options[:cql_index] && !definition.options[:solr_index]
|
14
15
|
unless index_exists?(model.column_family.to_s, attribute.to_s)
|
15
16
|
if index_exists?(model.column_family.to_s, attribute.to_s)
|
16
17
|
count += 1
|
17
|
-
say "Dropping solr index on #{attribute
|
18
|
+
say "Dropping solr index on #{attribute}", :subitem
|
18
19
|
DatastaxRails::Cql::DropIndex.new(solr_index_cql_name(model.column_family.to_s, attribute.to_s)).execute
|
19
20
|
end
|
20
21
|
count += 1
|
21
|
-
say "Creating cassandra index on #{attribute
|
22
|
+
say "Creating cassandra index on #{attribute}", :subitem
|
22
23
|
DatastaxRails::Cql::CreateIndex.new(cassandra_index_cql_name(model.column_family.to_s, attribute.to_s)).on(model.column_family.to_s).column(attribute.to_s).execute
|
23
24
|
end
|
24
|
-
elsif
|
25
|
-
unless column_exists?(model.column_family.to_s, "__#{attribute
|
25
|
+
elsif definition.options[:cql_index]
|
26
|
+
unless column_exists?(model.column_family.to_s, "__#{attribute}")
|
26
27
|
# Create and populate the new column
|
27
28
|
count += 1
|
28
29
|
say "Adding column '__#{attribute}'", :subitem
|
29
|
-
DatastaxRails::Cql::AlterColumnFamily.new(model.column_family).add("__#{attribute
|
30
|
+
DatastaxRails::Cql::AlterColumnFamily.new(model.column_family).add("__#{attribute}" => definition.cql_type).execute
|
30
31
|
say "Populating column '__#{attribute}' (this might take a while)", :subitem
|
31
|
-
export = "echo \"copy #{model.column_family
|
32
|
-
import = "echo \"copy #{model.column_family
|
32
|
+
export = "echo \"copy #{model.column_family} (key, #{attribute}) TO 'dsr_export.csv';\" | cqlsh #{model.current_server}"
|
33
|
+
import = "echo \"copy #{model.column_family} (key, __#{attribute}) FROM 'dsr_export.csv';\" | cqlsh #{model.current_server}"
|
33
34
|
if system(export)
|
34
35
|
system(import)
|
35
36
|
else
|
@@ -37,101 +38,101 @@ module DatastaxRails
|
|
37
38
|
end
|
38
39
|
end
|
39
40
|
count += 1
|
40
|
-
say "Creating cassandra index on __#{attribute
|
41
|
-
DatastaxRails::Cql::CreateIndex.new(cassandra_index_cql_name(model.column_family.to_s, "__#{attribute
|
41
|
+
say "Creating cassandra index on __#{attribute}", :subitem
|
42
|
+
DatastaxRails::Cql::CreateIndex.new(cassandra_index_cql_name(model.column_family.to_s, "__#{attribute}")).on(model.column_family.to_s).column("__#{attribute}").execute
|
42
43
|
end
|
43
44
|
end
|
44
45
|
count
|
45
46
|
end
|
46
|
-
|
47
|
+
|
47
48
|
# Creates a CQL3 backed column family
|
48
49
|
def create_cql3_column_family(model)
|
49
|
-
say
|
50
|
+
say 'Creating Column Family via CQL3', :subitem
|
50
51
|
columns = {}
|
51
|
-
model.attribute_definitions.each {|k,col| columns[k] = col.cql_type}
|
52
|
+
model.attribute_definitions.each { |k, col| columns[k] = col.cql_type }
|
52
53
|
pk = model.primary_key.to_s
|
53
|
-
if
|
54
|
-
pk += ", #{model.cluster_by
|
54
|
+
if model.respond_to?(:cluster_by) && model.cluster_by.present?
|
55
|
+
pk += ", #{model.cluster_by}"
|
55
56
|
end
|
56
57
|
cql = DatastaxRails::Cql::CreateColumnFamily.new(model.column_family).primary_key(pk).columns(columns)
|
57
58
|
cql.with(model.create_options) if model.create_options
|
58
59
|
cql.execute
|
59
60
|
end
|
60
|
-
|
61
|
+
|
61
62
|
# Creates the named keyspace
|
62
63
|
def create_keyspace(keyspace, options = {})
|
63
|
-
opts = { :
|
64
|
-
:
|
64
|
+
opts = { name: keyspace.to_s,
|
65
|
+
strategy_class: 'org.apache.cassandra.locator.NetworkTopologyStrategy' }.with_indifferent_access.merge(options)
|
65
66
|
|
66
|
-
if
|
67
|
-
say "Keyspace #{keyspace
|
67
|
+
if keyspace_exists?(keyspace.to_s)
|
68
|
+
say "Keyspace #{keyspace} already exists"
|
68
69
|
return false
|
69
70
|
else
|
70
71
|
cql = DatastaxRails::Cql::CreateKeyspace.new(opts.delete(:name))
|
71
72
|
cql.strategy_class(opts.delete(:strategy_class))
|
72
73
|
strategy_options = opts.delete('strategy_options')
|
73
74
|
cql.strategy_options(strategy_options.symbolize_keys)
|
74
|
-
say "Creating keyspace #{keyspace
|
75
|
+
say "Creating keyspace #{keyspace}"
|
75
76
|
cql.execute
|
76
77
|
return true
|
77
78
|
end
|
78
79
|
end
|
79
|
-
|
80
|
+
|
80
81
|
def drop_keyspace
|
81
|
-
say "Dropping keyspace #{@keyspace
|
82
|
+
say "Dropping keyspace #{@keyspace}"
|
82
83
|
DatastaxRails::Cql::DropKeyspace.new(@keyspace.to_s).execute
|
83
84
|
end
|
84
|
-
|
85
|
+
|
85
86
|
# Computes the expected solr index name as reported by CQL.
|
86
87
|
def solr_index_cql_name(cf, column)
|
87
|
-
"#{@keyspace}_#{cf
|
88
|
+
"#{@keyspace}_#{cf}_#{column}_index"
|
88
89
|
end
|
89
|
-
|
90
|
+
|
90
91
|
# Computes the expected cassandra index name as reported by CQL.
|
91
92
|
def cassandra_index_cql_name(cf, column)
|
92
|
-
"#{cf
|
93
|
+
"#{cf}_#{column}_idx"
|
93
94
|
end
|
94
|
-
|
95
|
+
|
95
96
|
# Checks the Cassandra system tables to see if a keyspace exists
|
96
97
|
def keyspace_exists?(keyspace)
|
97
|
-
klass = OpenStruct.new(:
|
98
|
+
klass = OpenStruct.new(column_family: 'system.schema_keyspaces', default_consistency: 'QUORUM')
|
98
99
|
cql = DatastaxRails::Cql::ColumnFamily.new(klass)
|
99
|
-
results = cql.select(
|
100
|
+
results = cql.select('count(*)').conditions('keyspace_name' => keyspace).execute
|
100
101
|
results.first['count'].to_i > 0
|
101
102
|
end
|
102
|
-
|
103
|
+
|
103
104
|
# Checks the Cassandra system tables to see if a column family exists
|
104
105
|
def column_family_exists?(cf)
|
105
|
-
klass = OpenStruct.new(:
|
106
|
+
klass = OpenStruct.new(column_family: 'system.schema_columnfamilies', default_consistency: 'QUORUM')
|
106
107
|
cql = DatastaxRails::Cql::ColumnFamily.new(klass)
|
107
|
-
results = cql.select(
|
108
|
+
results = cql.select('count(*)').conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf).execute
|
108
109
|
results.first['count'] > 0
|
109
110
|
end
|
110
|
-
|
111
|
+
|
111
112
|
# Checks the Cassandra system tables to see if a column exists on a column family
|
112
113
|
def column_exists?(cf, col)
|
113
|
-
klass = OpenStruct.new(:
|
114
|
+
klass = OpenStruct.new(column_family: 'system.schema_columns', default_consistency: 'QUORUM')
|
114
115
|
cql = DatastaxRails::Cql::ColumnFamily.new(klass)
|
115
|
-
results = cql.select(
|
116
|
+
results = cql.select('count(*)').conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf, 'column_name' => col).execute
|
116
117
|
exists = results.first['count'] > 0
|
117
118
|
unless exists
|
118
119
|
# We need to check if it's part of an alias (ugh)
|
119
|
-
klass = OpenStruct.new(:
|
120
|
+
klass = OpenStruct.new(column_family: 'system.schema_columnfamilies', default_consistency: 'QUORUM')
|
120
121
|
cql = DatastaxRails::Cql::ColumnFamily.new(klass)
|
121
|
-
results = cql.select(
|
122
|
+
results = cql.select('column_aliases, key_aliases, value_alias').conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf).execute
|
122
123
|
row = results.first
|
123
124
|
exists = row['key_aliases'].include?(col.to_s) || row['column_aliases'].include?(col.to_s) || (row['value_alias'] && row['value_alias'].include?(col.to_s))
|
124
125
|
end
|
125
126
|
exists
|
126
127
|
end
|
127
|
-
|
128
|
+
|
128
129
|
# Checks the Cassandra system tables to see if an index exists on a column family
|
129
130
|
def index_exists?(cf, col)
|
130
|
-
klass = OpenStruct.new(:
|
131
|
+
klass = OpenStruct.new(column_family: 'system.schema_columns', default_consistency: 'QUORUM')
|
131
132
|
cql = DatastaxRails::Cql::ColumnFamily.new(klass)
|
132
|
-
results = cql.select(
|
133
|
+
results = cql.select('index_name').conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf, 'column_name' => col).execute
|
133
134
|
results.first['index_name'] != nil
|
134
135
|
end
|
135
136
|
end
|
136
137
|
end
|
137
|
-
end
|
138
|
+
end
|
@@ -1,13 +1,15 @@
|
|
1
1
|
module DatastaxRails
|
2
2
|
module Schema
|
3
|
+
# DatastaxRails reads the attributes from the individual models. This class migrates both Cassandra
|
4
|
+
# and Solr to the point where they reflect what is specified in the models.
|
3
5
|
class Migrator
|
4
6
|
include DatastaxRails::Schema::Solr
|
5
7
|
include DatastaxRails::Schema::Cassandra
|
6
|
-
|
8
|
+
|
7
9
|
cattr_accessor :verbose
|
8
10
|
self.verbose = true
|
9
11
|
attr_accessor :errors
|
10
|
-
|
12
|
+
|
11
13
|
def initialize(keyspace)
|
12
14
|
@keyspace = keyspace
|
13
15
|
check_schema_migrations unless keyspace == 'system'
|
@@ -15,7 +17,7 @@ module DatastaxRails
|
|
15
17
|
end
|
16
18
|
|
17
19
|
def migrate_all(force = false)
|
18
|
-
say_with_time(
|
20
|
+
say_with_time('Migrating all models') do
|
19
21
|
|
20
22
|
FileList[rails_models].each do |model|
|
21
23
|
require model
|
@@ -23,14 +25,12 @@ module DatastaxRails
|
|
23
25
|
|
24
26
|
count = 0
|
25
27
|
DatastaxRails::Base.models.each do |m|
|
26
|
-
|
27
|
-
count += migrate_one(m, force)
|
28
|
-
end
|
28
|
+
count += migrate_one(m, force) unless m.abstract_class?
|
29
29
|
end
|
30
30
|
count
|
31
31
|
end
|
32
32
|
end
|
33
|
-
|
33
|
+
|
34
34
|
def migrate_one(model, force = false)
|
35
35
|
count = 0
|
36
36
|
say_with_time("Migrating #{model.name} to latest version") do
|
@@ -38,9 +38,9 @@ module DatastaxRails
|
|
38
38
|
create_cql3_column_family(model)
|
39
39
|
count += 1
|
40
40
|
end
|
41
|
-
|
41
|
+
|
42
42
|
count += check_missing_schema(model)
|
43
|
-
|
43
|
+
|
44
44
|
unless model <= DatastaxRails::CassandraOnlyModel
|
45
45
|
count += upload_solr_configuration(model, force)
|
46
46
|
end
|
@@ -54,49 +54,49 @@ module DatastaxRails
|
|
54
54
|
|
55
55
|
private
|
56
56
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
57
|
+
# Determine all models to be included within the migration
|
58
|
+
# using Rails config paths instead of absolute paths.
|
59
|
+
# This enables Rails Engines to monkey patch their own
|
60
|
+
# models in, to be automatically included within migrations.
|
61
|
+
#
|
62
|
+
# @see http://pivotallabs.com/leave-your-migrations-in-your-rails-engines/
|
63
|
+
#
|
64
|
+
# @return [Array] list of configured application models
|
65
|
+
def rails_models
|
66
|
+
Rails.configuration.paths['app/models'].expanded.map { |p| p + '/*.rb' }
|
67
|
+
end
|
68
|
+
|
69
|
+
# Checks to ensure that the schema_migrations column family exists and creates it if not
|
70
|
+
def check_schema_migrations
|
71
|
+
return if column_family_exists?('schema_migrations')
|
72
|
+
say 'Creating schema_migrations column family'
|
73
|
+
DatastaxRails::Cql::CreateColumnFamily.new('schema_migrations').primary_key('cf')
|
74
|
+
.columns(cf: :text, digest: :text, solrconfig: :text, stopwords: :text).execute
|
75
|
+
end
|
76
|
+
|
77
|
+
def write(text = '')
|
78
|
+
puts(text) if verbose
|
79
|
+
end
|
80
|
+
|
81
|
+
def say(message, subitem = false)
|
82
|
+
write "#{subitem ? ' ->' : '--'} #{message}"
|
83
|
+
end
|
84
|
+
|
85
|
+
def say_with_time(message)
|
86
|
+
say(message)
|
87
|
+
result = nil
|
88
|
+
time = Benchmark.measure { result = yield }
|
89
|
+
say format('%.4fs', time.real), :subitem
|
90
|
+
say("#{result} changes", :subitem) if result.is_a?(Integer)
|
91
|
+
result
|
92
|
+
end
|
93
|
+
|
94
|
+
def suppress_messages
|
95
|
+
save, self.verbose = verbose, false
|
96
|
+
yield
|
97
|
+
ensure
|
98
|
+
self.verbose = save
|
99
|
+
end
|
100
100
|
end
|
101
101
|
end
|
102
102
|
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# rubocop:disable Style/LineLength
|
1
2
|
module DatastaxRails
|
2
3
|
module Schema
|
3
4
|
module Solr
|
@@ -5,7 +6,7 @@ module DatastaxRails
|
|
5
6
|
# most normal circumstances for indexing. When a customized template is required, it can
|
6
7
|
# be placed in the application's config/solr directory using the naming convention
|
7
8
|
# column_family-schema.xml.erb. It will be processed as a normal ERB file. See the DSR version
|
8
|
-
# for examples.
|
9
|
+
# for examples.
|
9
10
|
def generate_solr_schema(model)
|
10
11
|
@fields = []
|
11
12
|
@copy_fields = []
|
@@ -15,32 +16,34 @@ module DatastaxRails
|
|
15
16
|
else
|
16
17
|
@primary_key = model.primary_key
|
17
18
|
end
|
18
|
-
@custom_fields =
|
19
|
+
@custom_fields = ''
|
19
20
|
@columns = model.attribute_definitions.values
|
20
|
-
@fields.sort! {|a,b| a[:name] <=> b[:name]}
|
21
|
-
@copy_fields.sort! {|a,b| a[:source] <=> b[:source]}
|
21
|
+
@fields.sort! { |a, b| a[:name] <=> b[:name] }
|
22
|
+
@copy_fields.sort! { |a, b| a[:source] <=> b[:source] }
|
22
23
|
@fulltext_fields.sort!
|
23
|
-
|
24
|
-
if Rails.root.join('config','solr',"#{model.column_family}-schema.xml.erb").exist?
|
24
|
+
|
25
|
+
if Rails.root.join('config', 'solr', "#{model.column_family}-schema.xml.erb").exist?
|
25
26
|
say "Using custom schema for #{model.name}", :subitem
|
26
|
-
ERB.new(Rails.root.join('config','solr',"#{model.column_family}-schema.xml.erb").read, 0, '>')
|
27
|
-
|
27
|
+
ERB.new(Rails.root.join('config', 'solr', "#{model.column_family}-schema.xml.erb").read, 0, '>')
|
28
|
+
.result(binding)
|
29
|
+
elsif Rails.root.join('config', 'solr', 'application-schema.xml.erb').exist?
|
28
30
|
say 'Using application default schema', :subitem
|
29
|
-
ERB.new(Rails.root.join('config','solr','application-schema.xml.erb').read, 0, '>').result(binding)
|
31
|
+
ERB.new(Rails.root.join('config', 'solr', 'application-schema.xml.erb').read, 0, '>').result(binding)
|
30
32
|
else
|
31
|
-
ERB.new(File.read(File.join(File.dirname(__FILE__),
|
33
|
+
ERB.new(File.read(File.join(File.dirname(__FILE__), '..', '..', '..', 'config', 'schema.xml.erb')), 0, '>')
|
34
|
+
.result(binding)
|
32
35
|
end
|
33
36
|
end
|
34
|
-
|
37
|
+
|
35
38
|
# Sends a command to Solr instructing it to reindex the data. The data is reindexed in the background,
|
36
39
|
# and the new index is swapped in once it is finished.
|
37
|
-
def reindex_solr(model, destructive=false)
|
38
|
-
url = "#{DatastaxRails::Base.solr_base_url}/admin/cores?action=RELOAD&name=#{DatastaxRails::Base.config[:keyspace]}.#{model.column_family}&reindex=true&deleteAll=#{destructive
|
40
|
+
def reindex_solr(model, destructive = false)
|
41
|
+
url = "#{DatastaxRails::Base.solr_base_url}/admin/cores?action=RELOAD&name=#{DatastaxRails::Base.config[:keyspace]}.#{model.column_family}&reindex=true&deleteAll=#{destructive}"
|
39
42
|
say "Posting reindex command to '#{url}'", :subitem
|
40
43
|
`curl -s -X POST '#{url}'`
|
41
|
-
say
|
44
|
+
say 'Reindexing will run in the background', :subitem
|
42
45
|
end
|
43
|
-
|
46
|
+
|
44
47
|
# Creates the initial Solr Core. This is required once the first time a Solr schema is uploaded.
|
45
48
|
# It will cause the data to be indexed in the background.
|
46
49
|
def create_solr_core(model)
|
@@ -48,97 +51,102 @@ module DatastaxRails
|
|
48
51
|
say "Posting create command to '#{url}'", :subitem
|
49
52
|
`curl -s -X POST '#{url}'`
|
50
53
|
end
|
51
|
-
|
54
|
+
|
52
55
|
# Uploads the necessary configuration files for solr to function
|
53
56
|
# The solrconfig and stopwords files can be overridden on a per-model basis
|
54
57
|
# by creating a file called config/solr/column_family-solrconfig.xml or
|
55
58
|
# config/solr/column_family-stopwords.txt
|
56
59
|
#
|
57
60
|
# TODO: find a way to upload arbitrary files automatically (e.g., additional stopwords lists)
|
58
|
-
|
61
|
+
# TODO: Simplify this method
|
62
|
+
def upload_solr_configuration(model, force = false, reindex = true) # rubocop:disable all
|
59
63
|
count = 0
|
60
|
-
if Rails.root.join('config','solr',"#{model.column_family}-solrconfig.xml").exist?
|
61
|
-
say
|
62
|
-
solrconfig = Rails.root.join('config','solr',"#{model.column_family}-solrconfig.xml").read
|
64
|
+
if Rails.root.join('config', 'solr', "#{model.column_family}-solrconfig.xml").exist?
|
65
|
+
say 'Using custom solrconfig file', :subitem
|
66
|
+
solrconfig = Rails.root.join('config', 'solr', "#{model.column_family}-solrconfig.xml").read
|
63
67
|
else
|
64
68
|
@legacy = model.legacy_mapping?
|
65
|
-
solrconfig = ERB.new(File.read(File.join(File.dirname(__FILE__),
|
69
|
+
solrconfig = ERB.new(File.read(File.join(File.dirname(__FILE__), '..', '..', '..', 'config', 'solrconfig.xml.erb'))).result(binding)
|
66
70
|
end
|
67
|
-
if Rails.root.join('config','solr',"#{model.column_family}-stopwords.txt").exist?
|
68
|
-
say
|
69
|
-
stopwords = Rails.root.join('config','solr',"#{model.column_family}-stopwords.txt").read
|
71
|
+
if Rails.root.join('config', 'solr', "#{model.column_family}-stopwords.txt").exist?
|
72
|
+
say 'Using custom stopwords file', :subitem
|
73
|
+
stopwords = Rails.root.join('config', 'solr', "#{model.column_family}-stopwords.txt").read
|
70
74
|
else
|
71
|
-
stopwords = File.read(File.join(File.dirname(__FILE__),
|
75
|
+
stopwords = File.read(File.join(File.dirname(__FILE__), '..', '..', '..', 'config', 'stopwords.txt'))
|
72
76
|
end
|
73
77
|
schema = generate_solr_schema(model)
|
74
78
|
solrconfig_digest = Digest::SHA1.hexdigest(solrconfig)
|
75
79
|
stopwords_digest = Digest::SHA1.hexdigest(stopwords)
|
76
80
|
schema_digest = Digest::SHA1.hexdigest(schema)
|
77
|
-
|
81
|
+
|
78
82
|
newcf = !column_exists?(model.column_family, 'solr_query')
|
79
83
|
force ||= newcf
|
80
|
-
|
81
|
-
results = DatastaxRails::Cql::Select.new(SchemaMigration, ['*']).conditions(:
|
84
|
+
|
85
|
+
results = DatastaxRails::Cql::Select.new(SchemaMigration, ['*']).conditions(cf: model.column_family).execute
|
82
86
|
sm_digests = results.first || {}
|
83
|
-
|
87
|
+
|
84
88
|
solr_url = "#{DatastaxRails::Base.solr_base_url}/resource/#{@keyspace}.#{model.column_family}"
|
85
|
-
|
89
|
+
|
86
90
|
uri = URI.parse(solr_url)
|
87
91
|
http = Net::HTTP.new(uri.host, uri.port)
|
88
92
|
if uri.scheme == 'https'
|
89
93
|
http.use_ssl = true
|
90
|
-
http.cert = OpenSSL::X509::Certificate.new(Rails.root.join(
|
91
|
-
http.key = OpenSSL::PKey::RSA.new(Rails.root.join(
|
92
|
-
http.ca_path = Rails.root.join(
|
94
|
+
http.cert = OpenSSL::X509::Certificate.new(Rails.root.join('config', 'datastax_rails.crt').read)
|
95
|
+
http.key = OpenSSL::PKey::RSA.new(Rails.root.join('config', 'datastax_rails.key').read)
|
96
|
+
http.ca_path = Rails.root.join('config', 'sade_ca.crt').to_s
|
93
97
|
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
94
98
|
end
|
95
99
|
http.read_timeout = 300
|
96
|
-
|
100
|
+
|
97
101
|
if force || solrconfig_digest != sm_digests['solrconfig']
|
98
102
|
count += 1
|
99
|
-
loop do
|
103
|
+
loop do
|
100
104
|
say "Posting Solr Config file to '#{solr_url}/solrconfig.xml'", :subitem
|
101
|
-
http.post(uri.path+
|
105
|
+
http.post(uri.path + '/solrconfig.xml', solrconfig)
|
102
106
|
if Rails.env.production?
|
103
107
|
sleep(5)
|
104
|
-
resp = http.get(uri.path+
|
105
|
-
continue unless resp.message == 'OK'
|
108
|
+
resp = http.get(uri.path + '/solrconfig.xml')
|
109
|
+
continue unless resp.message == 'OK'
|
106
110
|
end
|
107
111
|
break
|
108
112
|
end
|
109
|
-
DatastaxRails::Cql::Update.new(SchemaMigration, :
|
113
|
+
DatastaxRails::Cql::Update.new(SchemaMigration, cf: model.column_family).columns(solrconfig: solrconfig_digest).execute
|
110
114
|
end
|
111
115
|
if force || stopwords_digest != sm_digests['stopwords']
|
112
116
|
count += 1
|
113
117
|
loop do
|
114
118
|
say "Posting Solr Stopwords file to '#{solr_url}/stopwords.txt'", :subitem
|
115
|
-
http.post(uri.path+
|
119
|
+
http.post(uri.path + '/stopwords.txt', stopwords)
|
116
120
|
if Rails.env.production?
|
117
121
|
sleep(5)
|
118
|
-
resp = http.get(uri.path+
|
122
|
+
resp = http.get(uri.path + '/stopwords.txt')
|
119
123
|
continue unless resp.message == 'OK'
|
120
124
|
end
|
121
125
|
break
|
122
126
|
end
|
123
|
-
DatastaxRails::Cql::Update.new(SchemaMigration, :
|
127
|
+
DatastaxRails::Cql::Update.new(SchemaMigration, cf: model.column_family).columns(stopwords: stopwords_digest).execute
|
124
128
|
end
|
125
129
|
if force || schema_digest != sm_digests['digest']
|
126
130
|
count += 1
|
127
131
|
loop do
|
128
132
|
say "Posting Solr Schema file to '#{solr_url}/schema.xml'", :subitem
|
129
|
-
http.post(uri.path+
|
133
|
+
http.post(uri.path + '/schema.xml', schema)
|
130
134
|
if Rails.env.production?
|
131
135
|
sleep(5)
|
132
|
-
resp = http.get(uri.path+
|
136
|
+
resp = http.get(uri.path + '/schema.xml')
|
133
137
|
continue unless resp.message == 'OK'
|
134
138
|
end
|
135
139
|
break
|
136
140
|
end
|
137
|
-
DatastaxRails::Cql::Update.new(SchemaMigration, :
|
138
|
-
newcf
|
141
|
+
DatastaxRails::Cql::Update.new(SchemaMigration, cf: model.column_family).columns(digest: schema_digest).execute
|
142
|
+
if newcf
|
143
|
+
create_solr_core(model)
|
144
|
+
elsif reindex
|
145
|
+
reindex_solr(model)
|
146
|
+
end
|
139
147
|
end
|
140
148
|
count
|
141
149
|
end
|
142
150
|
end
|
143
151
|
end
|
144
|
-
end
|
152
|
+
end
|