datastax_rails 1.1.0.3 → 1.2.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (92) hide show
  1. checksums.yaml +4 -4
  2. data/README.rdoc +13 -13
  3. data/Rakefile +1 -0
  4. data/config/schema.xml.erb +0 -1
  5. data/config/{solrconfig.xml → solrconfig.xml.erb} +1 -1
  6. data/lib/blankslate.rb +1 -1
  7. data/lib/datastax_rails/associations/collection_proxy.rb +6 -2
  8. data/lib/datastax_rails/attribute_assignment.rb +114 -0
  9. data/lib/datastax_rails/attribute_methods/definition.rb +8 -2
  10. data/lib/datastax_rails/attribute_methods/typecasting.rb +2 -5
  11. data/lib/datastax_rails/attribute_methods.rb +9 -7
  12. data/lib/datastax_rails/base.rb +127 -109
  13. data/lib/datastax_rails/callbacks.rb +11 -7
  14. data/lib/datastax_rails/cassandra_only_model.rb +27 -0
  15. data/lib/datastax_rails/collection.rb +3 -1
  16. data/lib/datastax_rails/cql/base.rb +4 -0
  17. data/lib/datastax_rails/cql/select.rb +12 -2
  18. data/lib/datastax_rails/identity/abstract_key_factory.rb +1 -0
  19. data/lib/datastax_rails/identity/custom_key_factory.rb +1 -0
  20. data/lib/datastax_rails/identity/natural_key_factory.rb +1 -0
  21. data/lib/datastax_rails/identity/uuid_key_factory.rb +4 -0
  22. data/lib/datastax_rails/identity.rb +2 -1
  23. data/lib/datastax_rails/inheritance.rb +61 -0
  24. data/lib/datastax_rails/payload_model.rb +2 -5
  25. data/lib/datastax_rails/persistence.rb +63 -20
  26. data/lib/datastax_rails/railtie.rb +5 -1
  27. data/lib/datastax_rails/relation/batches.rb +2 -2
  28. data/lib/datastax_rails/relation/facet_methods.rb +56 -5
  29. data/lib/datastax_rails/relation/finder_methods.rb +55 -1
  30. data/lib/datastax_rails/relation/search_methods.rb +103 -32
  31. data/lib/datastax_rails/relation/spawn_methods.rb +3 -1
  32. data/lib/datastax_rails/relation/stats_methods.rb +1 -1
  33. data/lib/datastax_rails/relation.rb +166 -30
  34. data/lib/datastax_rails/schema/cassandra.rb +165 -0
  35. data/lib/datastax_rails/schema/migrator.rb +85 -193
  36. data/lib/datastax_rails/schema/solr.rb +158 -0
  37. data/lib/datastax_rails/schema.rb +2 -30
  38. data/lib/datastax_rails/scoping/default.rb +142 -0
  39. data/lib/datastax_rails/scoping/named.rb +200 -0
  40. data/lib/datastax_rails/scoping.rb +106 -349
  41. data/lib/datastax_rails/tasks/ds.rake +41 -42
  42. data/lib/datastax_rails/types/array_type.rb +1 -1
  43. data/lib/datastax_rails/types/base_type.rb +2 -2
  44. data/lib/datastax_rails/types/binary_type.rb +1 -1
  45. data/lib/datastax_rails/types/boolean_type.rb +1 -1
  46. data/lib/datastax_rails/types/date_type.rb +1 -1
  47. data/lib/datastax_rails/types/float_type.rb +4 -4
  48. data/lib/datastax_rails/types/integer_type.rb +3 -3
  49. data/lib/datastax_rails/types/string_type.rb +1 -1
  50. data/lib/datastax_rails/types/text_type.rb +1 -1
  51. data/lib/datastax_rails/types/time_type.rb +3 -3
  52. data/lib/datastax_rails/validations/uniqueness.rb +1 -1
  53. data/lib/datastax_rails/version.rb +1 -1
  54. data/lib/datastax_rails/wide_storage_model.rb +44 -0
  55. data/lib/datastax_rails.rb +16 -18
  56. data/spec/datastax_rails/associations_spec.rb +7 -3
  57. data/spec/datastax_rails/attribute_methods_spec.rb +23 -0
  58. data/spec/datastax_rails/base_spec.rb +1 -6
  59. data/spec/datastax_rails/inheritance_spec.rb +41 -0
  60. data/spec/datastax_rails/persistence_spec.rb +13 -3
  61. data/spec/datastax_rails/relation/batches_spec.rb +1 -1
  62. data/spec/datastax_rails/relation/facet_methods_spec.rb +52 -0
  63. data/spec/datastax_rails/relation/finder_methods_spec.rb +22 -1
  64. data/spec/datastax_rails/relation/search_methods_spec.rb +51 -1
  65. data/spec/datastax_rails/relation_spec.rb +14 -3
  66. data/spec/datastax_rails/schema/migrator_spec.rb +92 -0
  67. data/spec/datastax_rails/schema/solr_spec.rb +34 -0
  68. data/spec/datastax_rails/scoping/default_spec.rb +17 -0
  69. data/spec/datastax_rails/types/float_type_spec.rb +5 -9
  70. data/spec/datastax_rails/types/integer_type_spec.rb +5 -9
  71. data/spec/datastax_rails/types/time_type_spec.rb +28 -0
  72. data/spec/datastax_rails/validations/uniqueness_spec.rb +3 -1
  73. data/spec/dummy/config/application.rb +1 -4
  74. data/spec/dummy/config/datastax.yml +1 -1
  75. data/spec/dummy/config/environments/test.rb +2 -0
  76. data/spec/dummy/config/solr/articles-schema.xml.erb +1 -0
  77. data/spec/dummy/db/test.sqlite3 +0 -0
  78. data/spec/dummy/log/development.log +2 -0
  79. data/spec/dummy/log/production.log +2 -0
  80. data/spec/dummy/log/test.log +523 -0
  81. data/spec/spec_helper.rb +11 -0
  82. data/spec/support/models.rb +14 -0
  83. metadata +66 -22
  84. data/lib/datastax_rails/log_subscriber.rb +0 -37
  85. data/lib/datastax_rails/migrations/migration.rb +0 -15
  86. data/lib/datastax_rails/migrations.rb +0 -36
  87. data/lib/datastax_rails/mocking.rb +0 -15
  88. data/lib/datastax_rails/schema/migration.rb +0 -106
  89. data/lib/datastax_rails/schema/migration_proxy.rb +0 -25
  90. data/lib/datastax_rails/tasks/column_family.rb +0 -329
  91. data/lib/datastax_rails/tasks/keyspace.rb +0 -57
  92. data/spec/support/connection_double.rb +0 -6
@@ -0,0 +1,165 @@
1
+ module DatastaxRails
2
+ module Schema
3
+ module Cassandra
4
+ # Check for missing columns or columns needing cassandra indexes
5
+ def check_missing_schema(model)
6
+ count = 0
7
+ model.attribute_definitions.each do |attribute, definition|
8
+ unless column_exists?(model.column_family.to_s, attribute.to_s)
9
+ count += 1
10
+ say "Adding column '#{attribute}'", :subitem
11
+ DatastaxRails::Cql::AlterColumnFamily.new(model.column_family).add(attribute => :text).execute
12
+ end
13
+ if(definition.indexed == :cassandra)
14
+ unless index_exists?(model.column_family.to_s, attribute.to_s)
15
+ if index_exists?(model.column_family.to_s, attribute.to_s)
16
+ count += 1
17
+ say "Dropping solr index on #{attribute.to_s}", :subitem
18
+ DatastaxRails::Cql::DropIndex.new(solr_index_cql_name(model.column_family.to_s, attribute.to_s)).execute
19
+ end
20
+ count += 1
21
+ say "Creating cassandra index on #{attribute.to_s}", :subitem
22
+ DatastaxRails::Cql::CreateIndex.new(cassandra_index_cql_name(model.column_family.to_s, attribute.to_s)).on(model.column_family.to_s).column(attribute.to_s).execute
23
+ end
24
+ elsif(definition.indexed == :both)
25
+ unless column_exists?(model.column_family.to_s, "__#{attribute.to_s}")
26
+ # Create and populate the new column
27
+ count += 1
28
+ say "Adding column '__#{attribute}'", :subitem
29
+ DatastaxRails::Cql::AlterColumnFamily.new(model.column_family).add("__#{attribute.to_s}" => model.attribute_definitions[attribute].coder.options[:cassandra_type]).execute
30
+ say "Populating column '__#{attribute}' (this might take a while)", :subitem
31
+ export = "echo \"copy #{model.column_family.to_s} (key, #{attribute.to_s}) TO 'dsr_export.csv';\" | cqlsh #{model.current_server}"
32
+ import = "echo \"copy #{model.column_family.to_s} (key, __#{attribute.to_s}) FROM 'dsr_export.csv';\" | cqlsh #{model.current_server}"
33
+ if system(export)
34
+ system(import)
35
+ else
36
+ @errors << "Looks like you don't have a working cqlsh command in your path.\nRun the following two commands from a server with cqlsh:\n\n#{export}\n#{import}"
37
+ end
38
+ end
39
+ count += 1
40
+ say "Creating cassandra index on __#{attribute.to_s}", :subitem
41
+ DatastaxRails::Cql::CreateIndex.new(cassandra_index_cql_name(model.column_family.to_s, "__#{attribute.to_s}")).on(model.column_family.to_s).column("__#{attribute.to_s}").execute
42
+ end
43
+ end
44
+ count
45
+ end
46
+
47
+ # Creates a payload column family via CQL
48
+ def create_payload_column_family(model)
49
+ say "Creating Payload Column Family", :subitem
50
+ columns = {:chunk => :int, :payload => :text}
51
+ DatastaxRails::Cql::CreateColumnFamily.new(model.column_family).key_name(:digest).key_columns("digest, chunk").key_type(:text).columns(columns).with("COMPACT STORAGE").execute
52
+ end
53
+
54
+ # Creates a wide-storage column family via CQL
55
+ def create_wide_storage_column_family(model)
56
+ say "Creating Wide-Storage Column Family", :subitem
57
+ key_name = model.key_factory.attributes.join
58
+ cluster_by = model.cluster_by.keys.first
59
+ cluster_dir = model.cluster_by.values.first
60
+ key_columns = "#{key_name}, #{cluster_by}"
61
+ columns = {}
62
+ model.attribute_definitions.each {|k,v| columns[k] = v.coder.options[:cassandra_type] unless k.to_s == key_name}
63
+ DatastaxRails::Cql::CreateColumnFamily.new(model.column_family).key_name(key_name).key_columns(key_columns).key_type(:text).columns(columns).
64
+ with("CLUSTERING ORDER BY (#{cluster_by} #{cluster_dir.to_s.upcase})").execute
65
+ end
66
+
67
+ # Creates a regular cassandra-only column family via CQL
68
+ def create_cassandra_column_family(model)
69
+ say "Creating Cassandra-Only Column Family", :subitem
70
+ key_name = "key"
71
+ key_columns = "#{key_name}"
72
+ columns = {}
73
+ model.attribute_definitions.each {|k,v| columns[k] = v.coder.options[:cassandra_type]}
74
+ DatastaxRails::Cql::CreateColumnFamily.new(model.column_family).key_name(key_name).key_columns(key_columns).key_type(:text).columns(columns).execute
75
+ end
76
+
77
+ # Creates the named keyspace
78
+ def create_keyspace(keyspace, options = {})
79
+ opts = { :name => keyspace.to_s,
80
+ :strategy_class => 'org.apache.cassandra.locator.NetworkTopologyStrategy'}.with_indifferent_access.merge(options)
81
+
82
+ if(connection.keyspaces.collect(&:name).include?(keyspace.to_s))
83
+ say "Keyspace #{keyspace.to_s} already exists"
84
+ return false
85
+ else
86
+ cql = DatastaxRails::Cql::CreateKeyspace.new(opts.delete(:name))
87
+ cql.strategy_class(opts.delete(:strategy_class))
88
+ strategy_options = opts.delete('strategy_options')
89
+ cql.strategy_options(strategy_options.symbolize_keys)
90
+ say "Creating keyspace #{keyspace.to_s}"
91
+ cql.execute
92
+ return true
93
+ end
94
+ end
95
+
96
+ def drop_keyspace
97
+ say "Dropping keyspace #{@keyspace.to_s}"
98
+ DatastaxRails::Cql::DropKeyspace.new(@keyspace.to_s).execute
99
+ end
100
+
101
+ # Checks the Cassandra system tables to see if the key column is named properly. This is
102
+ # a migration method to handle the fact that Solr used to create column families with "KEY"
103
+ # instead of the now default "key".
104
+ def check_key_name(cf)
105
+ count = 0
106
+ if(cf.respond_to?(:column_family))
107
+ cf = cf.column_family
108
+ end
109
+ klass = OpenStruct.new(:column_family => 'system.schema_columnfamilies', :default_consistency => 'QUORUM')
110
+ cql = DatastaxRails::Cql::ColumnFamily.new(klass)
111
+ results = CassandraCQL::Result.new(cql.select("key_alias, key_aliases").conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf).execute)
112
+ result = results.fetch
113
+ if(result && (result['key_alias'] == 'KEY' || result['key_aliases'].include?('KEY')) && (result['key_aliases'].blank? || !result['key_aliases'].include?('key')))
114
+ count += 1
115
+ say "Renaming KEY column", :subitem
116
+ DatastaxRails::Cql::AlterColumnFamily.new(cf).rename("KEY",'key').execute
117
+ end
118
+ count
119
+ end
120
+
121
+ # Computes the expected solr index name as reported by CQL.
122
+ def solr_index_cql_name(cf, column)
123
+ "#{@keyspace}_#{cf.to_s}_#{column.to_s}_index"
124
+ end
125
+
126
+ # Computes the expected cassandra index name as reported by CQL.
127
+ def cassandra_index_cql_name(cf, column)
128
+ "#{cf.to_s}_#{column.to_s}_idx"
129
+ end
130
+
131
+ # Checks the Cassandra system tables to see if a column family exists
132
+ def column_family_exists?(cf)
133
+ klass = OpenStruct.new(:column_family => 'system.schema_columnfamilies', :default_consistency => 'QUORUM')
134
+ cql = DatastaxRails::Cql::ColumnFamily.new(klass)
135
+ results = CassandraCQL::Result.new(cql.select("count(*)").conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf).execute)
136
+ results.fetch['count'] > 0
137
+ end
138
+
139
+ # Checks the Cassandra system tables to see if a column exists on a column family
140
+ def column_exists?(cf, col)
141
+ klass = OpenStruct.new(:column_family => 'system.schema_columns', :default_consistency => 'QUORUM')
142
+ cql = DatastaxRails::Cql::ColumnFamily.new(klass)
143
+ results = CassandraCQL::Result.new(cql.select("count(*)").conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf, 'column_name' => col).execute)
144
+ exists = results.fetch['count'] > 0
145
+ unless exists
146
+ # We need to check if it's part of the primary key (ugh)
147
+ klass = OpenStruct.new(:column_family => 'system.schema_columnfamilies', :default_consistency => 'QUORUM')
148
+ cql = DatastaxRails::Cql::ColumnFamily.new(klass)
149
+ results = CassandraCQL::Result.new(cql.select("column_aliases, key_aliases").conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf).execute)
150
+ row = results.fetch
151
+ exists = row['key_aliases'].include?(col.to_s) || row['column_aliases'].include?(col.to_s)
152
+ end
153
+ exists
154
+ end
155
+
156
+ # Checks the Cassandra system tables to see if an index exists on a column family
157
+ def index_exists?(cf, col)
158
+ klass = OpenStruct.new(:column_family => 'system.schema_columns', :default_consistency => 'QUORUM')
159
+ cql = DatastaxRails::Cql::ColumnFamily.new(klass)
160
+ results = CassandraCQL::Result.new(cql.select("index_name").conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf, 'column_name' => col).execute)
161
+ results.fetch['index_name'] != nil
162
+ end
163
+ end
164
+ end
165
+ end
@@ -1,212 +1,104 @@
1
1
  module DatastaxRails
2
2
  module Schema
3
-
4
3
  class Migrator
5
-
6
- def self.migrate(migrations_path, target_version = nil)
7
- case
8
- when target_version.nil?
9
- up(migrations_path, target_version)
10
- when current_version == 0 && target_version == 0
11
- when current_version > target_version
12
- down(migrations_path, target_version)
13
- else
14
- up(migrations_path, target_version)
4
+ include DatastaxRails::Schema::Solr
5
+ include DatastaxRails::Schema::Cassandra
6
+
7
+ cattr_accessor :verbose
8
+ self.verbose = true
9
+ attr_accessor :errors
10
+
11
+ def initialize(keyspace)
12
+ @keyspace = keyspace
13
+ check_schema_migrations unless keyspace == 'system'
14
+ @errors = []
15
+ end
16
+
17
+ def migrate_all(force = false)
18
+ say_with_time("Migrating all models") do
19
+ # Ensure all models are loaded (necessary for non-production mode)
20
+ Dir[Rails.root.join("app","models",'*.rb').to_s].each do |file|
21
+ require File.basename(file, File.extname(file))
22
+ end
23
+ count = 0
24
+ DatastaxRails::Base.models.each do |m|
25
+ if !m.abstract_class?
26
+ count += migrate_one(m, force)
27
+ end
28
+ end
29
+ count
15
30
  end
16
31
  end
17
-
18
- def self.rollback(migrations_path, steps = 1)
19
- move(:down, migrations_path, steps)
20
- end
21
-
22
- def self.forward(migrations_path, steps = 1)
23
- move(:up, migrations_path, steps)
24
- end
25
-
26
- def self.up(migrations_path, target_version = nil)
27
- new(:up, migrations_path, target_version).migrate
28
- end
29
-
30
- def self.down(migrations_path, target_version = nil)
31
- new(:down, migrations_path, target_version).migrate
32
- end
33
-
34
- def self.run(direction, migrations_path, target_version)
35
- new(direction, migrations_path, target_version).run
36
- end
37
-
38
- def self.migrations_path
39
- 'ks/migrate'
40
- end
41
-
42
- def self.schema_migrations_column_family
43
- :schema_migrations
44
- end
45
-
46
- def self.column_family_tasks
47
- cas = DatastaxRails::Base.connection
48
- Tasks::ColumnFamily.new(cas.keyspace)
49
- end
50
-
51
- def self.get_all_versions
52
- cas = DatastaxRails::Base.connection
53
- cas.get(schema_migrations_column_family, 'all').map {|(name, _value)| name.to_i}.sort
54
- end
55
-
56
- def self.current_version
57
- sm_cf = schema_migrations_column_family
58
- if column_family_tasks.exists?(sm_cf)
59
- get_all_versions.max || 0
60
- else
61
- 0
32
+
33
+ def migrate_one(model, force = false)
34
+ count = 0
35
+ say_with_time("Migrating #{model.name} to latest version") do
36
+ if model.payload_model?
37
+ unless column_family_exists?(model.column_family.to_s)
38
+ create_payload_column_family(model)
39
+ count += 1
40
+ end
41
+ elsif model.wide_storage_model?
42
+ unless column_family_exists?(model.column_family.to_s)
43
+ create_wide_storage_column_family(model)
44
+ count += 1
45
+ end
46
+ count += check_missing_schema(model)
47
+ elsif model <= DatastaxRails::CassandraOnlyModel
48
+ unless column_family_exists?(model.column_family.to_s)
49
+ create_cassandra_column_family(model)
50
+ count += 1
51
+ end
52
+ count += check_key_name(model)
53
+ count += check_missing_schema(model)
54
+ else
55
+ count += check_key_name(model)
56
+ count += upload_solr_configuration(model, force)
57
+ count += check_missing_schema(model)
58
+ end
62
59
  end
60
+ count
63
61
  end
64
62
 
65
- private
66
-
67
- def self.move(direction, migrations_path, steps)
68
- migrator = self.new(direction, migrations_path)
69
- start_index = migrator.migrations.index(migrator.current_migration)
70
-
71
- if start_index
72
- finish = migrator.migrations[start_index + steps]
73
- version = finish ? finish.version : 0
74
- send(direction, migrations_path, version)
75
- end
63
+ def connection
64
+ DatastaxRails::Base.connection
76
65
  end
77
66
 
78
- public
79
-
80
- def initialize(direction, migrations_path, target_version = nil)
81
- sm_cf = self.class.schema_migrations_column_family
82
-
83
- unless column_family_tasks.exists?(sm_cf)
84
- column_family_tasks.create(sm_cf) do |cf|
85
- cf.comparator_type = 'LongType'
67
+ private
68
+
69
+ # Checks to ensure that the schema_migrations column family exists and creates it if not
70
+ def check_schema_migrations
71
+ unless column_family_exists?('schema_migrations')
72
+ say "Creating schema_migrations column family"
73
+ connection.execute_cql_query(DatastaxRails::Cql::CreateColumnFamily.new('schema_migrations').key_type(:text).columns(:digest => :text, :solrconfig => :text, :stopwords => :text).to_cql)
86
74
  end
75
+
76
+ check_key_name('schema_migrations')
87
77
  end
88
-
89
- @direction, @migrations_path, @target_version = direction, migrations_path, target_version
90
- end
91
-
92
- def current_version
93
- migrated.last || 0
94
- end
95
-
96
- def current_migration
97
- migrations.detect { |m| m.version == current_version }
98
- end
99
-
100
- def run
101
- target = migrations.detect { |m| m.version == @target_version }
102
- raise UnknownMigrationVersionError.new(@target_version) if target.nil?
103
- unless (up? && migrated.include?(target.version.to_i)) || (down? && !migrated.include?(target.version.to_i))
104
- target.migrate(@direction)
105
- record_version_state_after_migrating(target)
78
+
79
+ def write(text="")
80
+ puts(text) if verbose
106
81
  end
107
- end
108
-
109
- def migrate
110
- current = migrations.detect { |m| m.version == current_version }
111
- target = migrations.detect { |m| m.version == @target_version }
112
-
113
- if target.nil? && !@target_version.nil? && @target_version > 0
114
- raise UnknownMigrationVersionError.new(@target_version)
82
+
83
+ def say(message, subitem=false)
84
+ write "#{subitem ? " ->" : "--"} #{message}"
115
85
  end
116
-
117
- start = up? ? 0 : (migrations.index(current) || 0)
118
- finish = migrations.index(target) || migrations.size - 1
119
- runnable = migrations[start..finish]
120
-
121
- # skip the last migration if we're headed down, but not ALL the way down
122
- runnable.pop if down? && !target.nil?
123
-
124
- runnable.each do |migration|
125
- #puts "Migrating to #{migration.name} (#{migration.version})"
126
-
127
- # On our way up, we skip migrating the ones we've already migrated
128
- next if up? && migrated.include?(migration.version.to_i)
129
-
130
- # On our way down, we skip reverting the ones we've never migrated
131
- if down? && !migrated.include?(migration.version.to_i)
132
- migration.announce 'never migrated, skipping'; migration.write
133
- next
134
- end
135
-
136
- migration.migrate(@direction)
137
- record_version_state_after_migrating(migration)
86
+
87
+ def say_with_time(message)
88
+ say(message)
89
+ result = nil
90
+ time = Benchmark.measure { result = yield }
91
+ say "%.4fs" % time.real, :subitem
92
+ say("#{result} changes", :subitem) if result.is_a?(Integer)
93
+ result
138
94
  end
139
- end
140
-
141
- def migrations
142
- @migrations ||= begin
143
- files = Dir["#{@migrations_path}/[0-9]*_*.rb"]
144
-
145
- migrations = files.inject([]) do |klasses, file|
146
- version, name = file.scan(/([0-9]+)_([_a-z0-9]*).rb/).first
147
-
148
- raise IllegalMigrationNameError.new(file) unless version
149
- version = version.to_i
150
-
151
- if klasses.detect { |m| m.version == version }
152
- raise DuplicateMigrationVersionError.new(version)
153
- end
154
-
155
- if klasses.detect { |m| m.name == name.camelize }
156
- raise DuplicateMigrationNameError.new(name.camelize)
157
- end
158
-
159
- migration = MigrationProxy.new
160
- migration.name = name.camelize
161
- migration.version = version
162
- migration.filename = file
163
- klasses << migration
164
- end
165
-
166
- migrations = migrations.sort_by { |m| m.version }
167
- down? ? migrations.reverse : migrations
168
- end
169
- end
170
-
171
- def pending_migrations
172
- already_migrated = migrated
173
- migrations.reject { |m| already_migrated.include?(m.version.to_i) }
174
- end
175
-
176
- def migrated
177
- @migrated_versions ||= self.class.get_all_versions
178
- end
179
-
180
- private
181
-
182
- def column_family_tasks
183
- Tasks::ColumnFamily.new(connection.keyspace)
184
- end
185
-
186
- def connection
187
- DatastaxRails::Base.connection
188
- end
189
-
190
- def record_version_state_after_migrating(migration)
191
- sm_cf = self.class.schema_migrations_column_family
192
-
193
- @migrated_versions ||= []
194
- if down?
195
- @migrated_versions.delete(migration.version)
196
- connection.remove sm_cf, 'all', migration.version
197
- else
198
- @migrated_versions.push(migration.version).sort!
199
- connection.insert sm_cf, 'all', { migration.version => migration.name }
95
+
96
+ def suppress_messages
97
+ save, self.verbose = verbose, false
98
+ yield
99
+ ensure
100
+ self.verbose = save
200
101
  end
201
- end
202
-
203
- def up?
204
- @direction == :up
205
- end
206
-
207
- def down?
208
- @direction == :down
209
- end
210
102
  end
211
103
  end
212
104
  end
@@ -0,0 +1,158 @@
1
+ module DatastaxRails
2
+ module Schema
3
+ module Solr
4
+ # Generates a SOLR schema file. The default schema template included with DSR can handle
5
+ # most normal circumstances for indexing. When a customized template is required, it can
6
+ # be placed in the application's config/solr directory using the naming convention
7
+ # column_family-schema.xml.erb. It will be processed as a normal ERB file. See the DSR version
8
+ # for examples.
9
+ def generate_solr_schema(model)
10
+ @fields = []
11
+ @copy_fields = []
12
+ @fulltext_fields = []
13
+ @custom_fields = ""
14
+ model.attribute_definitions.values.each do |attr|
15
+ coder = attr.coder
16
+ if coder.options[:solr_type]
17
+ @fields.push({ :name => attr.name,
18
+ :type => coder.options[:solr_type].to_s,
19
+ :indexed => (coder.options[:indexed] == :solr || coder.options[:indexed] == :both).to_s,
20
+ :stored => coder.options[:stored].to_s,
21
+ :multi_valued => coder.options[:multi_valued].to_s })
22
+ end
23
+ if coder.options[:sortable] && coder.options[:tokenized]
24
+ @fields.push({ :name => "sort_" + attr.name,
25
+ :type => "string",
26
+ :indexed => true,
27
+ :stored => false,
28
+ :multi_valued => false })
29
+ @copy_fields.push({ :source => attr.name, :dest => "sort_" + attr.name }) if (coder.options[:indexed] || coder.options[:stored])
30
+ end
31
+ if coder.options[:fulltext]
32
+ @fulltext_fields << attr.name if (coder.options[:indexed] || coder.options[:stored])
33
+ end
34
+ end
35
+ # Sort the fields so that no matter what order the attributes are arranged into the
36
+ # same schema file gets generated
37
+ @fields.sort! {|a,b| a[:name] <=> b[:name]}
38
+ @copy_fields.sort! {|a,b| a[:source] <=> b[:source]}
39
+ @fulltext_fields.sort!
40
+
41
+ if Rails.root.join('config','solr',"#{model.column_family}-schema.xml.erb").exist?
42
+ say "Using custom schema for #{model.name}", :subitem
43
+ ERB.new(Rails.root.join('config','solr',"#{model.column_family}-schema.xml.erb").read).result(binding)
44
+ else
45
+ ERB.new(File.read(File.join(File.dirname(__FILE__),"..","..","..","config","schema.xml.erb"))).result(binding)
46
+ end
47
+ end
48
+
49
+ # Sends a command to Solr instructing it to reindex the data. The data is reindexed in the background,
50
+ # and the new index is swapped in once it is finished.
51
+ def reindex_solr(model)
52
+ url = "#{DatastaxRails::Base.solr_base_url}/admin/cores?action=RELOAD&name=#{DatastaxRails::Base.config[:keyspace]}.#{model.column_family}&reindex=true&deleteAll=false"
53
+ say "Posting reindex command to '#{url}'", :subitem
54
+ `curl -s -X POST '#{url}'`
55
+ say "Reindexing will run in the background", :subitem
56
+ end
57
+
58
+ # Creates the initial Solr Core. This is required once the first time a Solr schema is uploaded.
59
+ # It will cause the data to be indexed in the background.
60
+ def create_solr_core(model)
61
+ url = "#{DatastaxRails::Base.solr_base_url}/admin/cores?action=CREATE&name=#{DatastaxRails::Base.config[:keyspace]}.#{model.column_family}"
62
+ say "Posting create command to '#{url}'", :subitem
63
+ `curl -s -X POST '#{url}'`
64
+ end
65
+
66
+ # Uploads the necessary configuration files for solr to function
67
+ # The solrconfig and stopwords files can be overridden on a per-model basis
68
+ # by creating a file called config/solr/column_family-solrconfig.xml or
69
+ # config/solr/column_family-stopwords.txt
70
+ #
71
+ # TODO: find a way to upload arbitrary files automatically (e.g., additional stopwords lists)
72
+ def upload_solr_configuration(model, force=false)
73
+ count = 0
74
+ if Rails.root.join('config','solr',"#{model.column_family}-solrconfig.xml").exist?
75
+ say "Using custom solrconfig file", :subitem
76
+ solrconfig = Rails.root.join('config','solr',"#{model.column_family}-solrconfig.xml").read
77
+ else
78
+ @legacy = model.legacy_mapping?
79
+ solrconfig = ERB.new(File.read(File.join(File.dirname(__FILE__),"..","..","..","config","solrconfig.xml.erb"))).result(binding)
80
+ end
81
+ if Rails.root.join('config','solr',"#{model.column_family}-stopwords.txt").exist?
82
+ say "Using custom stopwords file", :subitem
83
+ stopwords = Rails.root.join('config','solr',"#{model.column_family}-stopwords.txt").read
84
+ else
85
+ stopwords = File.read(File.join(File.dirname(__FILE__),"..","..","..","config","stopwords.txt"))
86
+ end
87
+ schema = generate_solr_schema(model)
88
+ solrconfig_digest = Digest::SHA1.hexdigest(solrconfig)
89
+ stopwords_digest = Digest::SHA1.hexdigest(stopwords)
90
+ schema_digest = Digest::SHA1.hexdigest(schema)
91
+
92
+ newcf = !column_family_exists?(model.column_family.to_s)
93
+ force ||= newcf
94
+
95
+ results = DatastaxRails::Cql::Select.new(SchemaMigration, ['*']).conditions(:key => model.column_family).execute
96
+ sm_digests = CassandraCQL::Result.new(results).fetch.try(:to_hash) || {}
97
+
98
+ solr_url = "#{DatastaxRails::Base.solr_base_url}/resource/#{@keyspace}.#{model.column_family}"
99
+
100
+ uri = URI.parse(solr_url)
101
+ http = Net::HTTP.new(uri.host, uri.port)
102
+ if uri.scheme == 'https'
103
+ http.use_ssl = true
104
+ http.cert = OpenSSL::X509::Certificate.new(Rails.root.join("config","datastax_rails.crt").read)
105
+ http.key = OpenSSL::PKey::RSA.new(Rails.root.join("config","datastax_rails.key").read)
106
+ http.ca_path = Rails.root.join("config","sade_ca.crt").to_s
107
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
108
+ end
109
+ http.read_timeout = 300
110
+
111
+ if force || solrconfig_digest != sm_digests['solrconfig']
112
+ count += 1
113
+ loop do
114
+ say "Posting Solr Config file to '#{solr_url}/solrconfig.xml'", :subitem
115
+ http.post(uri.path+"/solrconfig.xml", solrconfig)
116
+ if Rails.env.production?
117
+ sleep(5)
118
+ resp = http.get(uri.path+"/solrconfig.xml")
119
+ continue unless resp.message == 'OK'
120
+ end
121
+ break
122
+ end
123
+ DatastaxRails::Cql::Update.new(SchemaMigration, model.column_family).columns(:solrconfig => solrconfig_digest).execute
124
+ end
125
+ if force || stopwords_digest != sm_digests['stopwords']
126
+ count += 1
127
+ loop do
128
+ say "Posting Solr Stopwords file to '#{solr_url}/stopwords.txt'", :subitem
129
+ http.post(uri.path+"/stopwords.txt", stopwords)
130
+ if Rails.env.production?
131
+ sleep(5)
132
+ resp = http.get(uri.path+"/stopwords.txt")
133
+ continue unless resp.message == 'OK'
134
+ end
135
+ break
136
+ end
137
+ DatastaxRails::Cql::Update.new(SchemaMigration, model.column_family).columns(:stopwords => stopwords_digest).execute
138
+ end
139
+ if force || schema_digest != sm_digests['digest']
140
+ count += 1
141
+ loop do
142
+ say "Posting Solr Schema file to '#{solr_url}/schema.xml'", :subitem
143
+ http.post(uri.path+"/schema.xml", schema)
144
+ if Rails.env.production?
145
+ sleep(5)
146
+ resp = http.get(uri.path+"/schema.xml")
147
+ continue unless resp.message == 'OK'
148
+ end
149
+ break
150
+ end
151
+ DatastaxRails::Cql::Update.new(SchemaMigration, model.column_family).columns(:digest => schema_digest).execute
152
+ newcf ? create_solr_core(model) : reindex_solr(model)
153
+ end
154
+ count
155
+ end
156
+ end
157
+ end
158
+ end
@@ -2,36 +2,8 @@ module DatastaxRails
2
2
  module Schema
3
3
  extend ActiveSupport::Autoload
4
4
 
5
- class IrreversibleMigration < StandardError
6
- end
7
-
8
- class DuplicateMigrationVersionError < StandardError#:nodoc:
9
- def initialize(version)
10
- super("Multiple migrations have the version number #{version}")
11
- end
12
- end
13
-
14
- class DuplicateMigrationNameError < StandardError#:nodoc:
15
- def initialize(name)
16
- super("Multiple migrations have the name #{name}")
17
- end
18
- end
19
-
20
- class UnknownMigrationVersionError < StandardError #:nodoc:
21
- def initialize(version)
22
- super("No migration with version number #{version}")
23
- end
24
- end
25
-
26
- class IllegalMigrationNameError < StandardError#:nodoc:
27
- def initialize(name)
28
- super("Illegal name for migration file: #{name}\n\t(only lower case letters, numbers, and '_' allowed)")
29
- end
30
- end
31
-
5
+ autoload :Solr
6
+ autoload :Cassandra
32
7
  autoload :Migrator
33
- autoload :Migration
34
- autoload :MigrationProxy
35
-
36
8
  end
37
9
  end