nobrainer 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/no_brainer/config.rb +42 -16
- data/lib/no_brainer/connection.rb +1 -0
- data/lib/no_brainer/connection_manager.rb +1 -1
- data/lib/no_brainer/criteria.rb +1 -1
- data/lib/no_brainer/criteria/aggregate.rb +8 -5
- data/lib/no_brainer/criteria/core.rb +8 -7
- data/lib/no_brainer/criteria/count.rb +1 -1
- data/lib/no_brainer/criteria/delete.rb +1 -1
- data/lib/no_brainer/criteria/extend.rb +31 -0
- data/lib/no_brainer/criteria/first.rb +1 -1
- data/lib/no_brainer/criteria/index.rb +1 -1
- data/lib/no_brainer/criteria/limit.rb +0 -1
- data/lib/no_brainer/criteria/order_by.rb +8 -9
- data/lib/no_brainer/criteria/pluck.rb +1 -1
- data/lib/no_brainer/criteria/raw.rb +1 -1
- data/lib/no_brainer/criteria/scope.rb +6 -6
- data/lib/no_brainer/criteria/update.rb +3 -3
- data/lib/no_brainer/criteria/where.rb +24 -32
- data/lib/no_brainer/document/association.rb +5 -5
- data/lib/no_brainer/document/association/belongs_to.rb +6 -6
- data/lib/no_brainer/document/association/core.rb +11 -11
- data/lib/no_brainer/document/association/eager_loader.rb +1 -1
- data/lib/no_brainer/document/association/has_many.rb +7 -7
- data/lib/no_brainer/document/association/has_many_through.rb +1 -1
- data/lib/no_brainer/document/association/has_one.rb +1 -1
- data/lib/no_brainer/document/atomic_ops.rb +26 -18
- data/lib/no_brainer/document/attributes.rb +9 -8
- data/lib/no_brainer/document/callbacks.rb +1 -1
- data/lib/no_brainer/document/core.rb +1 -1
- data/lib/no_brainer/document/criteria.rb +9 -2
- data/lib/no_brainer/document/dirty.rb +1 -3
- data/lib/no_brainer/document/index.rb +13 -79
- data/lib/no_brainer/document/index/index.rb +83 -0
- data/lib/no_brainer/document/index/meta_store.rb +31 -0
- data/lib/no_brainer/document/index/synchronizer.rb +68 -0
- data/lib/no_brainer/document/lazy_fetch.rb +5 -5
- data/lib/no_brainer/document/persistance.rb +27 -7
- data/lib/no_brainer/document/polymorphic.rb +1 -1
- data/lib/no_brainer/document/types.rb +6 -4
- data/lib/no_brainer/document/uniqueness.rb +3 -3
- data/lib/no_brainer/document/validation.rb +13 -4
- data/lib/no_brainer/fork.rb +1 -0
- data/lib/no_brainer/query_runner/database_on_demand.rb +6 -5
- data/lib/no_brainer/query_runner/logger.rb +10 -6
- data/lib/no_brainer/query_runner/missing_index.rb +5 -4
- data/lib/no_brainer/query_runner/reconnect.rb +20 -17
- data/lib/no_brainer/query_runner/run_options.rb +3 -0
- data/lib/no_brainer/query_runner/table_on_demand.rb +11 -8
- data/lib/no_brainer/railtie.rb +5 -5
- data/lib/no_brainer/railtie/database.rake +12 -12
- data/lib/no_brainer/rql.rb +9 -0
- data/lib/nobrainer.rb +5 -3
- metadata +8 -8
- data/lib/no_brainer/index_manager.rb +0 -9
@@ -0,0 +1,83 @@
|
|
1
|
+
class NoBrainer::Document::Index::Index < Struct.new(
|
2
|
+
:model, :name, :aliased_name, :kind, :what, :external, :geo, :multi, :meta)
|
3
|
+
|
4
|
+
MetaStore = NoBrainer::Document::Index::MetaStore
|
5
|
+
|
6
|
+
def initialize(*args)
|
7
|
+
super
|
8
|
+
|
9
|
+
self.name = self.name.to_sym
|
10
|
+
self.aliased_name = self.aliased_name.to_sym
|
11
|
+
self.external = !!self.external
|
12
|
+
self.geo = !!self.geo
|
13
|
+
self.multi = !!self.multi
|
14
|
+
end
|
15
|
+
|
16
|
+
def same_definition?(other)
|
17
|
+
# allow name to change through renames
|
18
|
+
self.model == other.model &&
|
19
|
+
self.geo == other.geo &&
|
20
|
+
self.multi == other.multi &&
|
21
|
+
self.serialized_rql_proc == other.serialized_rql_proc
|
22
|
+
end
|
23
|
+
|
24
|
+
def human_name
|
25
|
+
index_name = "index #{model}.#{name}"
|
26
|
+
index_name += " as #{aliased_name}" unless name == aliased_name
|
27
|
+
index_name
|
28
|
+
end
|
29
|
+
|
30
|
+
def rql_proc
|
31
|
+
case kind
|
32
|
+
when :single then ->(doc) { doc[model.lookup_field_alias(what)] }
|
33
|
+
when :compound then ->(doc) { what.map { |field| doc[model.lookup_field_alias(field)] } }
|
34
|
+
when :proc then what # TODO XXX not translating the field aliases
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def serialized_rql_proc
|
39
|
+
meta.try(:rql_function) || (rql_proc && NoBrainer::RQL.rql_proc_as_json(rql_proc))
|
40
|
+
end
|
41
|
+
|
42
|
+
def show_op(verb, options={})
|
43
|
+
color = case verb
|
44
|
+
when :create then "\e[1;32m" # green
|
45
|
+
when :delete then "\e[1;31m" # red
|
46
|
+
when :update then "\e[1;33m" # yellow
|
47
|
+
end
|
48
|
+
STDERR.puts "[NoBrainer] #{color}#{verb.to_s.capitalize} #{human_name}\e[0m" if options[:verbose]
|
49
|
+
end
|
50
|
+
|
51
|
+
def create(options={})
|
52
|
+
show_op(:create, options)
|
53
|
+
|
54
|
+
opt = {}
|
55
|
+
opt[:multi] = true if multi
|
56
|
+
opt[:geo] = true if geo
|
57
|
+
|
58
|
+
NoBrainer::RQL.reset_lambda_var_counter
|
59
|
+
NoBrainer.run(model.rql_table.index_create(aliased_name, opt, &rql_proc))
|
60
|
+
|
61
|
+
MetaStore.on(model.database_name) do
|
62
|
+
MetaStore.create(:table_name => model.table_name, :index_name => aliased_name,
|
63
|
+
:rql_function => serialized_rql_proc)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def delete(options={})
|
68
|
+
show_op(:delete, options)
|
69
|
+
|
70
|
+
NoBrainer.run(model.rql_table.index_drop(aliased_name))
|
71
|
+
|
72
|
+
MetaStore.on(model.database_name) do
|
73
|
+
MetaStore.where(:table_name => model.table_name, :index_name => aliased_name).delete_all
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def update(wanted_index, options={})
|
78
|
+
wanted_index.show_op(:update, options)
|
79
|
+
|
80
|
+
self.delete(options.merge(:verbose => false))
|
81
|
+
wanted_index.create(options.merge(:verbose => false))
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
class NoBrainer::Document::Index::MetaStore
|
2
|
+
include NoBrainer::Document
|
3
|
+
include NoBrainer::Document::Timestamps
|
4
|
+
|
5
|
+
disable_perf_warnings
|
6
|
+
|
7
|
+
default_scope ->{ order_by(:created_at) }
|
8
|
+
|
9
|
+
store_in :database => ->{ Thread.current[:nobrainer_meta_store_db] },
|
10
|
+
:table => 'nobrainer_index_meta'
|
11
|
+
|
12
|
+
field :table_name, :type => String, :required => true
|
13
|
+
field :index_name, :type => String, :required => true
|
14
|
+
field :rql_function, :type => String, :required => true
|
15
|
+
|
16
|
+
def rql_function=(value)
|
17
|
+
super(JSON.dump(value))
|
18
|
+
end
|
19
|
+
|
20
|
+
def rql_function
|
21
|
+
JSON.load(super)
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.on(db_name, &block)
|
25
|
+
old_db_name = Thread.current[:nobrainer_meta_store_db]
|
26
|
+
Thread.current[:nobrainer_meta_store_db] = db_name
|
27
|
+
NoBrainer.with(:auto_create_tables => true) { block.call }
|
28
|
+
ensure
|
29
|
+
Thread.current[:nobrainer_meta_store_db] = old_db_name
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
class NoBrainer::Document::Index::Synchronizer
|
2
|
+
Index = NoBrainer::Document::Index::Index
|
3
|
+
MetaStore = NoBrainer::Document::Index::MetaStore
|
4
|
+
|
5
|
+
def initialize(models)
|
6
|
+
@models_indexes_map = Hash[models.map do |model|
|
7
|
+
[model, model.indexes.values.reject { |index| index.name == model.pk_name }]
|
8
|
+
end]
|
9
|
+
end
|
10
|
+
|
11
|
+
def meta_store_on(db_name)
|
12
|
+
@meta_store ||= {}
|
13
|
+
@meta_store[db_name] ||= MetaStore.on(db_name) { MetaStore.all.to_a }
|
14
|
+
end
|
15
|
+
|
16
|
+
class Op < Struct.new(:index, :op, :args)
|
17
|
+
def run(options={})
|
18
|
+
index.__send__(op, *args, options)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def _generate_plan_for(model, wanted_indexes)
|
23
|
+
current_indexes = NoBrainer.run(model.rql_table.index_status).map do |s|
|
24
|
+
meta = meta_store_on(model.database_name)
|
25
|
+
.select { |i| i.table_name == model.table_name && i.index_name == s['index'] }.last
|
26
|
+
Index.new(model, s['index'], s['index'], nil, nil, nil, s['geo'], s['multi'], meta)
|
27
|
+
end
|
28
|
+
|
29
|
+
all_aliased_names = (wanted_indexes + current_indexes).map(&:aliased_name).uniq
|
30
|
+
all_aliased_names.map do |aliased_name|
|
31
|
+
wanted_index = wanted_indexes.select { |i| i.aliased_name == aliased_name }.first
|
32
|
+
current_index = current_indexes.select { |i| i.aliased_name == aliased_name }.first
|
33
|
+
|
34
|
+
next if wanted_index.try(:external)
|
35
|
+
|
36
|
+
case [!wanted_index.nil?, !current_index.nil?]
|
37
|
+
when [true, false] then Op.new(wanted_index, :create)
|
38
|
+
when [false, true] then Op.new(current_index, :delete)
|
39
|
+
when [true, true] then
|
40
|
+
case wanted_index.same_definition?(current_index)
|
41
|
+
when true then nil # up to date
|
42
|
+
when false then Op.new(current_index, :update, [wanted_index])
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end.compact
|
46
|
+
end
|
47
|
+
|
48
|
+
def generate_plan
|
49
|
+
@models_indexes_map.map { |model, indexes| _generate_plan_for(model, indexes) }.flatten(1)
|
50
|
+
end
|
51
|
+
|
52
|
+
def sync_indexes(options={})
|
53
|
+
plan = generate_plan
|
54
|
+
plan.each { |op| op.run(options) }
|
55
|
+
unless options[:wait] == false
|
56
|
+
models = plan.map(&:index).map(&:model).uniq
|
57
|
+
models.each { |model| NoBrainer.run(model.rql_table.index_wait()) }
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
class << self
|
62
|
+
def instance
|
63
|
+
new(NoBrainer::Document.all)
|
64
|
+
end
|
65
|
+
|
66
|
+
delegate :sync_indexes, :to => :instance
|
67
|
+
end
|
68
|
+
end
|
@@ -19,7 +19,7 @@ module NoBrainer::Document::LazyFetch
|
|
19
19
|
lazy_fetch = self.class.fields_to_lazy_fetch.to_a
|
20
20
|
return super unless lazy_fetch.present?
|
21
21
|
return super if options[:pluck]
|
22
|
-
super(options.
|
22
|
+
super(options.deep_merge(:without => lazy_fetch, :lazy_fetch => lazy_fetch))
|
23
23
|
end
|
24
24
|
|
25
25
|
module ClassMethods
|
@@ -31,12 +31,12 @@ module NoBrainer::Document::LazyFetch
|
|
31
31
|
def _field(attr, options={})
|
32
32
|
super
|
33
33
|
attr = attr.to_s
|
34
|
-
|
34
|
+
model = self
|
35
35
|
inject_in_layer :lazy_fetch do
|
36
36
|
if options[:lazy_fetch]
|
37
|
-
|
37
|
+
model.for_each_subclass { |_model| _model.fields_to_lazy_fetch << attr }
|
38
38
|
else
|
39
|
-
|
39
|
+
model.for_each_subclass { |_model| _model.fields_to_lazy_fetch.delete(attr) }
|
40
40
|
end
|
41
41
|
|
42
42
|
# Lazy loading can also specified through criteria.
|
@@ -57,7 +57,7 @@ module NoBrainer::Document::LazyFetch
|
|
57
57
|
|
58
58
|
def _remove_field(attr, options={})
|
59
59
|
super
|
60
|
-
for_each_subclass { |
|
60
|
+
for_each_subclass { |model| model.fields_to_lazy_fetch.delete(attr) }
|
61
61
|
inject_in_layer :lazy_fetch do
|
62
62
|
remove_method("#{attr}") if method_defined?("#{attr}")
|
63
63
|
end
|
@@ -60,7 +60,8 @@ module NoBrainer::Document::Persistance
|
|
60
60
|
end
|
61
61
|
|
62
62
|
def _create(options={})
|
63
|
-
return false if options[:validate] && !valid?
|
63
|
+
return false if options[:validate] != false && !valid?(nil, :clear_errors => false)
|
64
|
+
|
64
65
|
attrs = self.class.persistable_attributes(@_attributes, :instance => self)
|
65
66
|
result = NoBrainer.run(self.class.rql_table.insert(attrs))
|
66
67
|
self.pk_value ||= result['generated_keys'].to_a.first
|
@@ -74,7 +75,7 @@ module NoBrainer::Document::Persistance
|
|
74
75
|
end
|
75
76
|
|
76
77
|
def _update_only_changed_attrs(options={})
|
77
|
-
return false if options[:validate] && !valid?
|
78
|
+
return false if options[:validate] != false && !valid?(nil, :clear_errors => false)
|
78
79
|
|
79
80
|
# We won't be using the `changes` values, because they went through
|
80
81
|
# read_attribute(), and we want the raw values.
|
@@ -89,23 +90,42 @@ module NoBrainer::Document::Persistance
|
|
89
90
|
true
|
90
91
|
end
|
91
92
|
|
92
|
-
def
|
93
|
-
options = options.reverse_merge(:validate => true)
|
93
|
+
def _save?(options)
|
94
94
|
new_record? ? _create(options) : _update_only_changed_attrs(options)
|
95
95
|
end
|
96
96
|
|
97
|
+
def save?(options={})
|
98
|
+
errors.clear
|
99
|
+
_save?(options)
|
100
|
+
end
|
101
|
+
|
97
102
|
def save(*args)
|
98
103
|
save?(*args) or raise NoBrainer::Error::DocumentInvalid, self
|
104
|
+
nil
|
99
105
|
end
|
100
106
|
|
101
|
-
def
|
107
|
+
def save!(*args)
|
108
|
+
save(*args)
|
109
|
+
:you_should_be_using_the_non_bang_version_of_save
|
110
|
+
end
|
111
|
+
|
112
|
+
def update?(attrs, options={})
|
102
113
|
assign_attributes(attrs, options)
|
103
114
|
save?(options)
|
104
115
|
end
|
116
|
+
alias_method :update_attributes?, :update?
|
117
|
+
|
118
|
+
def update(*args)
|
119
|
+
update?(*args) or raise NoBrainer::Error::DocumentInvalid, self
|
120
|
+
nil
|
121
|
+
end
|
122
|
+
alias_method :update_attributes, :update
|
105
123
|
|
106
|
-
def
|
107
|
-
|
124
|
+
def update!(*args)
|
125
|
+
update(*args)
|
126
|
+
:you_should_be_using_the_non_bang_version_of_update
|
108
127
|
end
|
128
|
+
alias_method :update_attributes!, :update!
|
109
129
|
|
110
130
|
def delete
|
111
131
|
unless @destroyed
|
@@ -60,6 +60,8 @@ module NoBrainer::Document::Types
|
|
60
60
|
def _field(attr, options={})
|
61
61
|
super
|
62
62
|
|
63
|
+
return unless options[:type]
|
64
|
+
|
63
65
|
NoBrainer::Document::Types.load_type_extensions(options[:type]) if options[:type]
|
64
66
|
|
65
67
|
inject_in_layer :types do
|
@@ -105,13 +107,13 @@ module NoBrainer::Document::Types
|
|
105
107
|
class << self
|
106
108
|
mattr_accessor :loaded_extensions
|
107
109
|
self.loaded_extensions = Set.new
|
108
|
-
def load_type_extensions(
|
109
|
-
unless loaded_extensions.include?(
|
110
|
+
def load_type_extensions(model)
|
111
|
+
unless loaded_extensions.include?(model)
|
110
112
|
begin
|
111
|
-
require File.join(File.dirname(__FILE__), 'types',
|
113
|
+
require File.join(File.dirname(__FILE__), 'types', model.name.underscore)
|
112
114
|
rescue LoadError
|
113
115
|
end
|
114
|
-
loaded_extensions <<
|
116
|
+
loaded_extensions << model
|
115
117
|
end
|
116
118
|
end
|
117
119
|
end
|
@@ -66,10 +66,10 @@ module NoBrainer::Document::Uniqueness
|
|
66
66
|
|
67
67
|
def initialize(options={})
|
68
68
|
super
|
69
|
-
|
69
|
+
model = options[:class]
|
70
70
|
self.scope = [*options[:scope]]
|
71
|
-
([
|
72
|
-
|
71
|
+
([model] + model.descendants).each do |_model|
|
72
|
+
_model.unique_validators << self
|
73
73
|
end
|
74
74
|
end
|
75
75
|
|
@@ -5,19 +5,28 @@ module NoBrainer::Document::Validation
|
|
5
5
|
|
6
6
|
included do
|
7
7
|
# We don't want before_validation returning false to halt the chain.
|
8
|
-
define_callbacks :validation, :skip_after_callbacks_if_terminated => true,
|
9
|
-
:terminator => proc { false }
|
8
|
+
define_callbacks :validation, :skip_after_callbacks_if_terminated => true,
|
9
|
+
:scope => [:kind, :name], :terminator => proc { false }
|
10
10
|
end
|
11
11
|
|
12
|
-
def valid?(context=nil)
|
13
|
-
|
12
|
+
def valid?(context=nil, options={})
|
13
|
+
context ||= new_record? ? :create : :update
|
14
|
+
|
15
|
+
# copy/pasted, because we need to have control on errors.clear
|
16
|
+
current_context, self.validation_context = validation_context, context
|
17
|
+
errors.clear unless options[:clear_errors] == false
|
18
|
+
run_validations!
|
19
|
+
ensure
|
20
|
+
self.validation_context = current_context
|
14
21
|
end
|
15
22
|
|
16
23
|
module ClassMethods
|
17
24
|
def _field(attr, options={})
|
18
25
|
super
|
26
|
+
validates(attr, { :format => { :with => options[:format] } }) if options.has_key?(:format)
|
19
27
|
validates(attr, { :presence => options[:required] }) if options.has_key?(:required)
|
20
28
|
validates(attr, { :uniqueness => options[:unique] }) if options.has_key?(:unique)
|
29
|
+
validates(attr, { :uniqueness => options[:uniq] }) if options.has_key?(:uniq)
|
21
30
|
validates(attr, { :inclusion => {:in => options[:in]} }) if options.has_key?(:in)
|
22
31
|
validates(attr, options[:validates]) if options[:validates]
|
23
32
|
end
|
data/lib/no_brainer/fork.rb
CHANGED
@@ -2,24 +2,25 @@ class NoBrainer::QueryRunner::DatabaseOnDemand < NoBrainer::QueryRunner::Middlew
|
|
2
2
|
def call(env)
|
3
3
|
@runner.call(env)
|
4
4
|
rescue RuntimeError => e
|
5
|
-
if database_name =
|
5
|
+
if database_name = handle_database_on_demand_exception?(env, e)
|
6
6
|
auto_create_database(env, database_name)
|
7
7
|
retry
|
8
8
|
end
|
9
9
|
raise
|
10
10
|
end
|
11
11
|
|
12
|
-
def
|
13
|
-
NoBrainer::Config.auto_create_databases
|
12
|
+
def handle_database_on_demand_exception?(env, e)
|
13
|
+
(NoBrainer::Config.auto_create_databases || env[:auto_create_databases]) &&
|
14
|
+
e.message =~ /^Database `(.+)` does not exist\.$/ && $1
|
14
15
|
end
|
15
16
|
|
16
17
|
private
|
17
18
|
|
18
19
|
def auto_create_database(env, database_name)
|
19
|
-
if env[:
|
20
|
+
if env[:last_auto_create_database] == database_name
|
20
21
|
raise "Auto database creation is not working with #{database_name}"
|
21
22
|
end
|
22
|
-
env[:
|
23
|
+
env[:last_auto_create_database] = database_name
|
23
24
|
|
24
25
|
NoBrainer.db_create(database_name)
|
25
26
|
rescue RuntimeError => e
|
@@ -10,8 +10,12 @@ class NoBrainer::QueryRunner::Logger < NoBrainer::QueryRunner::Middleware
|
|
10
10
|
private
|
11
11
|
|
12
12
|
def log_query(env, start_time, exception=nil)
|
13
|
-
return if
|
14
|
-
|
13
|
+
return if handle_on_demand_exception?(env, exception)
|
14
|
+
|
15
|
+
not_indexed = env[:criteria] && env[:criteria].where_present? &&
|
16
|
+
!env[:criteria].where_indexed? &&
|
17
|
+
!env[:criteria].model.try(:perf_warnings_disabled)
|
18
|
+
|
15
19
|
level = exception ? Logger::ERROR :
|
16
20
|
not_indexed ? Logger::INFO : Logger::DEBUG
|
17
21
|
return if NoBrainer.logger.nil? || NoBrainer.logger.level > level
|
@@ -19,7 +23,7 @@ class NoBrainer::QueryRunner::Logger < NoBrainer::QueryRunner::Middleware
|
|
19
23
|
duration = Time.now - start_time
|
20
24
|
|
21
25
|
msg_duration = (duration * 1000.0).round(1).to_s
|
22
|
-
msg_duration = " " * [0,
|
26
|
+
msg_duration = " " * [0, 6 - msg_duration.size].max + msg_duration
|
23
27
|
msg_duration = "[#{msg_duration}ms] "
|
24
28
|
|
25
29
|
msg_db = "[#{env[:db_name]}] " if env[:db_name] && env[:db_name].to_s != NoBrainer.connection.parsed_uri[:db]
|
@@ -49,9 +53,9 @@ class NoBrainer::QueryRunner::Logger < NoBrainer::QueryRunner::Middleware
|
|
49
53
|
NoBrainer.logger.add(level, msg)
|
50
54
|
end
|
51
55
|
|
52
|
-
def
|
56
|
+
def handle_on_demand_exception?(env, e)
|
53
57
|
# pretty gross I must say.
|
54
|
-
e && (NoBrainer::QueryRunner::DatabaseOnDemand.new(nil).
|
55
|
-
NoBrainer::QueryRunner::TableOnDemand.new(nil).
|
58
|
+
e && (NoBrainer::QueryRunner::DatabaseOnDemand.new(nil).handle_database_on_demand_exception?(env, e) ||
|
59
|
+
NoBrainer::QueryRunner::TableOnDemand.new(nil).handle_table_on_demand_exception?(env, e))
|
56
60
|
end
|
57
61
|
end
|