nobrainer 0.27.0 → 0.28.0

data/lib/no_brainer/document/missing_attributes.rb

@@ -6,9 +6,14 @@ module NoBrainer::Document::MissingAttributes
   end
 
   def assign_attributes(attrs, options={})
+    # there is one and only one key :pluck or :without to missing_attributes
     if options[:missing_attributes]
-      # there is one and only one key :pluck or :without to missing_attributes
-      @missing_attributes = options[:missing_attributes]
+      # TODO XXX this whole thing is gross.
+      # if @missing_attributes is already there, it's because we are doing a
+      # incremental reload. clear_missing_field will do the work of recognizing
+      # which fields are here or not.
+      @missing_attributes ||= options[:missing_attributes]
+
       assert_access_field(self.class.pk_name, "The primary key is not accessible. Use .raw or")
       assert_access_field(:_type, "The subclass type is not accessible. Use .raw or") if self.class.is_polymorphic
     end

data/lib/no_brainer/document/primary_key.rb

@@ -3,6 +3,7 @@ module NoBrainer::Document::PrimaryKey
   autoload :Generator
 
   extend ActiveSupport::Concern
+  include ActiveModel::Conversion
 
   DEFAULT_PK_NAME = :id
 
@@ -27,6 +28,11 @@ module NoBrainer::Document::PrimaryKey
     "#{self.class.table_name}/#{pk_value}"
   end
 
+  def to_key
+    # ActiveModel::Conversion
+    [pk_value]
+  end
+
   module ClassMethods
     def define_default_pk
       class_variable_set(:@@pk_name, nil)
@@ -51,19 +57,19 @@ module NoBrainer::Document::PrimaryKey
     end
 
     def field(attr, options={})
-      if options[:primary_key]
+      if attr.to_sym == pk_name || options[:primary_key]
         options = options.merge(:readonly => true) if options[:readonly].nil?
         options = options.merge(:index => true)
 
-        if options[:default].nil?
-          # TODO Maybe we should let the user configure the pk generator
-          default_pk_generator = NoBrainer::Document::PrimaryKey::Generator
-          if options[:type].in?([default_pk_generator.field_type, nil])
-            options[:type] = default_pk_generator.field_type
-            options[:default] = ->{ default_pk_generator.generate }
-          end
+        # TODO Maybe we should let the user configure the pk generator
+        pk_generator = NoBrainer::Document::PrimaryKey::Generator
+
+        if options[:type].in?([pk_generator.field_type, nil]) && !options.key?(:default)
+          options[:type] = pk_generator.field_type
+          options[:default] = ->{ pk_generator.generate }
         end
       end
+
       super
     end
 

data/lib/no_brainer/document/table_config.rb

@@ -0,0 +1,118 @@
+require 'rethinkdb'
+
+module NoBrainer::Document::TableConfig
+  extend ActiveSupport::Concern
+  extend NoBrainer::Autoload
+
+  autoload :Synchronizer
+
+  VALID_TABLE_CONFIG_OPTIONS = [:name, :durability, :shards, :replicas, :primary_replica_tag, :write_acks]
+
+  included do
+    cattr_accessor :table_config_options, :instance_accessor => false
+    self.table_config_options = {}
+  end
+
+  module ClassMethods
+    def store_in(options)
+      if options[:table]
+        STDERR.puts "[NoBrainer] `store_in(table: ...)' has been removed. Use `table_config(name: ...)' instead."
+        options[:name] = options.delete(:table)
+      end
+
+      if options[:database] || options[:db]
+        raise "`store_in(db: ...)' has been removed. Use `run_with(db: ...)' instead."
+      end
+
+      table_config(options)
+    end
+
+    def _set_table_config(options)
+      raise "table_config() must be used at the parent class, not a subclass" unless is_root_class?
+
+      options.assert_valid_keys(*VALID_TABLE_CONFIG_OPTIONS)
+      self.table_config_options.merge!(options)
+    end
+
+    def table_name
+      name = table_config_options[:name]
+      name = name.call if name.is_a?(Proc)
+      (name || root_class.name.tableize.gsub('/', '__')).to_s
+    end
+
+    def rql_table
+      RethinkDB::RQL.new.table(table_name)
+    end
+
+    def table_config(options={})
+      return _set_table_config(options) unless options.empty?
+      NoBrainer::System::TableConfig.new_from_db(NoBrainer.run { rql_table.config })
+    end
+
+    def table_status
+      NoBrainer::System::TableConfig.new_from_db(NoBrainer.run { rql_table.status })
+    end
+
+    def table_stats
+      NoBrainer::System::Stats.where(:db => NoBrainer.current_db, :table => table_name).to_a
+    end
+
+    def rebalance
+      NoBrainer.run { rql_table.rebalance }
+      true
+    end
+
+    def table_wait
+      NoBrainer.run { rql_table.wait }
+    end
+
+    def table_create_options
+      NoBrainer::Config.table_options
+        .merge(table_config_options)
+        .merge(:name => table_name)
+        .merge(:primary_key => lookup_field_alias(pk_name))
+        .reverse_merge(:durability => 'hard')
+        .reduce({}) { |h,(k,v)| h[k] = v.is_a?(Symbol) ? v.to_s : v; h } # symbols -> strings
+    end
+
+    def sync_table_config(options={})
+      c = table_create_options
+      table_config.update!(c.slice(:durability, :primary_key, :write_acks))
+      NoBrainer.run { rql_table.reconfigure(c.slice(:shards, :replicas, :primary_replica_tag)) }
+      true
+    end
+
+    def sync_indexes(options={})
+      NoBrainer::Document::Index::Synchronizer.new(self).sync_indexes(options)
+    end
+
+    def sync_schema(options={})
+      sync_table_config(options)
+      sync_indexes(options)
+    end
+  end
+
+  class << self
+    def sync_table_config(options={})
+      models = NoBrainer::Document.all(:types => [:user, :nobrainer])
+      NoBrainer::Document::TableConfig::Synchronizer.new(models).sync_table_config(options)
+    end
+
+    def sync_indexes(options={})
+      # nobrainer models don't have indexes
+      models = NoBrainer::Document.all(:types => [:user])
+      NoBrainer::Document::Index::Synchronizer.new(models).sync_indexes(options)
+    end
+
+    def sync_schema(options={})
+      sync_table_config(options)
+      sync_indexes(options)
+    end
+
+    def rebalance(options={})
+      models = NoBrainer::Document.all(:types => [:user])
+      models.each(&:rebalance)
+      true
+    end
+  end
+end

data/lib/no_brainer/document/table_config/synchronizer.rb

@@ -0,0 +1,21 @@
+class NoBrainer::Document::TableConfig::Synchronizer
+  def initialize(models)
+    @models = models
+  end
+
+  def sync_table_config(options={})
+    # XXX A bit funny since we might touch the lock table...
+    lock = NoBrainer::Lock.new('nobrainer:sync_table_config')
+
+    lock.synchronize do
+      @models.each(&:sync_table_config)
+    end
+
+    unless options[:wait] == false
+      # Waiting on all models due to possible races
+      @models.each(&:table_wait)
+    end
+
+    true
+  end
+end

data/lib/no_brainer/document/timestamps.rb

@@ -21,4 +21,8 @@ module NoBrainer::Document::Timestamps
   def cache_key
     "#{super}#{updated_at.try(:strftime, "-%s%L")}"
   end
+
+  def touch
+    update!(:updated_at => Time.now)
+  end
 end

data/lib/no_brainer/document/validation/core.rb

@@ -32,7 +32,7 @@ module NoBrainer::Document::Validation::Core
 
       shorthands = SHORTHANDS
       shorthands = shorthands.merge(:required => :not_null) if options[:type] == NoBrainer::Boolean
-      shorthands.each { |k,v| validates(attr, v => options[k]) if options.has_key?(k) }
+      shorthands.each { |k,v| validates(attr, v => options[k]) if options.key?(k) }
 
       validates(attr, options[:validates]) if options[:validates]
       validates(attr, :length => { :minimum => options[:min_length] }) if options[:min_length]

data/lib/no_brainer/document/validation/uniqueness.rb

@@ -17,7 +17,7 @@ module NoBrainer::Document::Validation::Uniqueness
   end
 
   def unlock_unique_fields
-    @locked_keys_for_uniqueness.to_h.values.each(&:unlock)
+    (@locked_keys_for_uniqueness || {}).values.each(&:unlock)
     @locked_keys_for_uniqueness = {}
   end
 

data/lib/no_brainer/lock.rb

@@ -3,7 +3,7 @@ require 'digest/sha1'
 class NoBrainer::Lock
   include NoBrainer::Document
 
-  store_in :table => 'nobrainer_locks'
+  table_config :name => 'nobrainer_locks'
 
   # Since PKs are limited to 127 characters, we can't use the user's key as a PK
   # as it could be arbitrarily long.
@@ -106,13 +106,13 @@ class NoBrainer::Lock
     end
   end
 
+  def save?(*);  raise NotImplementedError; end
+  def delete(*); raise NotImplementedError; end
+
   private
 
   def set_expiration(options)
     expire = NoBrainer::Config.lock_options.merge(options)[:expire]
     self.expires_at = RethinkDB::RQL.new.now + expire
   end
-
-  def save?; raise; end
-  def delete; raise; end
 end

data/lib/no_brainer/profiler/logger.rb

@@ -6,7 +6,7 @@ class NoBrainer::Profiler::Logger
 
     level = env[:exception] ? Logger::ERROR :
              not_indexed ? Logger::INFO : Logger::DEBUG
-    return if NoBrainer.logger.nil? || NoBrainer.logger.level > level
+    return if NoBrainer.logger.level > level
 
     msg_duration = (env[:duration] * 1000.0).round(1).to_s
     msg_duration = " " * [0, 6 - msg_duration.size].max + msg_duration

data/lib/no_brainer/query_runner/database_on_demand.rb

@@ -21,7 +21,7 @@ class NoBrainer::QueryRunner::DatabaseOnDemand < NoBrainer::QueryRunner::Middlew
     end
     env[:last_auto_create_database] = db_name
 
-    NoBrainer.db_create(db_name)
+    NoBrainer.run { |r| r.db_create(db_name) }
   rescue RuntimeError => e
     # We might have raced with another db_create
     raise unless e.message =~ /Database `#{db_name}` already exists/

data/lib/no_brainer/query_runner/reconnect.rb

@@ -2,28 +2,44 @@ class NoBrainer::QueryRunner::Reconnect < NoBrainer::QueryRunner::Middleware
   def call(env)
     @runner.call(env)
   rescue StandardError => e
-    context ||= { :retries => NoBrainer::Config.max_retries_on_connection_failure }
     if is_connection_error_exception?(e)
-      if NoBrainer::Config.max_retries_on_connection_failure == 0
-        NoBrainer.disconnect
-      else
-        # XXX Possibly dangerous, as we could reexecute a non idempotent operation
-        # Check the semantics of the db
-        retry if reconnect(e, context)
-      end
+      context ||= {}
+      # XXX Possibly dangerous, as we could reexecute a non idempotent operation
+      retry if reconnect(e, context)
     end
     raise
   end
 
   private
 
+  def max_tries
+    NoBrainer::Config.max_retries_on_connection_failure
+  end
+
   def reconnect(e, context)
-    return false if context[:retries].zero?
-    context[:retries] -= 1
+    context[:connection_retries] ||= max_tries
+    context[:previous_connection] ||= NoBrainer.connection
+    NoBrainer.disconnect
+
+    unless context[:lost_connection_logged]
+      context[:lost_connection_logged] = true
+
+      msg = server_not_ready?(e) ? "Server %s not ready: %s" : "Connection issue with %s: %s"
+      NoBrainer.logger.warn(msg % [context[:previous_connection].try(:uri), exception_msg(e)])
+    end
+
+    if context[:connection_retries].zero?
+      NoBrainer.logger.info("Retry limit exceeded (#{max_tries}). Giving up.")
+      return false
+    end
+    context[:connection_retries] -= 1
 
-    warn_reconnect(e)
     sleep 1
-    NoBrainer.connection.reconnect(:noreply_wait => false)
+
+    c = NoBrainer.connection
+    NoBrainer.logger.info("Connecting to #{c.uri}... (last error: #{exception_msg(e)})")
+    c.connect
+
     true
   rescue StandardError => e
     retry if is_connection_error_exception?(e)
@@ -36,20 +52,20 @@ class NoBrainer::QueryRunner::Reconnect < NoBrainer::QueryRunner::Middleware
       Errno::ECONNRESET, Errno::ETIMEDOUT, IOError
       true
     when RethinkDB::RqlRuntimeError
-      e.message =~ /Primary .* not available/ ||
+      e.message =~ /lost contact/ ||
+      e.message =~ /(P|p)rimary .* not available/||
       e.message =~ /Connection.*closed/
     else
       false
     end
   end
 
-  def warn_reconnect(e)
-    if e.is_a?(RethinkDB::RqlRuntimeError)
-      e_msg = e.message.split("\n").first
-      msg = "Server #{NoBrainer::Config.rethinkdb_url} not ready - #{e_msg}, retrying..."
-    else
-      msg = "Connection issue with #{NoBrainer::Config.rethinkdb_url} - #{e}, retrying..."
-    end
-    NoBrainer.logger.try(:warn, msg)
+  def exception_msg(e)
+    e.is_a?(RethinkDB::RqlRuntimeError) ? e.message.split("\n").first : e.to_s
+  end
+
+  def server_not_ready?(e)
+    e.message =~ /lost contact/ ||
+    e.message =~ /(P|p)rimary .* not available/
   end
 end

data/lib/no_brainer/query_runner/table_on_demand.rb

@@ -16,9 +16,8 @@ class NoBrainer::QueryRunner::TableOnDemand < NoBrainer::QueryRunner::Middleware
   private
 
   def auto_create_table(env, db_name, table_name)
-    model ||= NoBrainer::Document::Core._all.select { |m| m.table_name == table_name }.first
-    model ||= NoBrainer::Document::Core._all_nobrainer.select { |m| m.table_name == table_name }.first
-
+    model = NoBrainer::Document.all(:types => [:user, :nobrainer])
+                               .detect { |m| m.table_name == table_name }
     if model.nil?
       raise "Auto table creation is not working for `#{db_name}.#{table_name}` -- Can't find the corresponding model."
     end
@@ -28,8 +27,16 @@ class NoBrainer::QueryRunner::TableOnDemand < NoBrainer::QueryRunner::Middleware
     end
     env[:last_auto_create_table] = [db_name, table_name]
 
-    NoBrainer.run_with(:db => db_name) do
-      NoBrainer.table_create(table_name, :primary_key => model.lookup_field_alias(model.pk_name))
+    create_options = model.table_create_options
+
+    NoBrainer.run(:db => db_name) do |r|
+      r.table_create(table_name, create_options.reject { |k,_| k.in? [:name, :write_acks] })
+    end
+
+    if create_options[:write_acks] && create_options[:write_acks] != 'single'
+      NoBrainer.run(:db => db_name) do |r|
+        r.table(table_name).config().update(:write_acks => create_options[:write_acks])
+      end
     end
   rescue RuntimeError => e
     # We might have raced with another table create

data/lib/no_brainer/railtie/database.rake

@@ -9,8 +9,18 @@ namespace :nobrainer do
     NoBrainer.sync_indexes(:verbose => true)
   end
 
-  task :sync_indexes_quiet => :environment do
-    NoBrainer.sync_indexes
+  desc 'Synchronize table configuration'
+  task :sync_table_config => :environment do
+    NoBrainer.sync_table_config(:verbose => true)
+  end
+
+  desc 'Synchronize indexes and table configuration'
+  task :sync_schema => :environment do
+    NoBrainer.sync_schema(:verbose => true)
+  end
+
+  task :sync_schema_quiet => :environment do
+    NoBrainer.sync_schema
   end
 
   desc 'Load seed data from db/seeds.rb'
@@ -18,8 +28,8 @@ namespace :nobrainer do
     Rails.application.load_seed
   end
 
-  desc 'Equivalent to :sync_indexes_quiet + :seed'
-  task :setup => [:sync_indexes_quiet, :seed]
+  desc 'Equivalent to :sync_schema_quiet + :seed'
+  task :setup => [:sync_schema_quiet, :seed]
 
   desc 'Equivalent to :drop + :setup'
   task :reset => [:drop, :setup]

data/lib/no_brainer/rql.rb

@@ -21,8 +21,9 @@ module NoBrainer::RQL
     case rql.is_a?(RethinkDB::RQL) && rql.body.is_a?(Array) && rql.body.first
     when UPDATE, DELETE, REPLACE, INSERT
       :write
-    when DB_CREATE, DB_DROP, DB_LIST, TABLE_CREATE, TABLE_DROP, TABLE_LIST, SYNC,
-         INDEX_CREATE, INDEX_DROP, INDEX_LIST, INDEX_STATUS, INDEX_WAIT
+    when DB_CREATE, DB_DROP, DB_LIST, TABLE_CREATE, TABLE_DROP, TABLE_LIST,
+         INDEX_CREATE, INDEX_DROP, INDEX_LIST, INDEX_STATUS, INDEX_WAIT, INDEX_RENAME,
+         CONFIG, STATUS, WAIT, RECONFIGURE, REBALANCE, SYNC
       :management
     else
       # XXX Not necessarily correct, but we'll be happy for logging colors.