queris 0.8.1

data/lib/queris/query/timer.rb

@@ -0,0 +1,42 @@
+module Queris
+  class Query
+    class Timer
+      def initialize
+        @time_start={}
+        @time={}
+        @times_recorded={}
+      end
+      def start(attr)
+        attr = attr.to_sym
+        @time_start[attr]=Time.now.to_f
+        @time[attr] ||= '?'
+      end
+      def finish(attr)
+        attr = attr.to_sym
+        start_time = @time_start[attr]
+        raise "Query Profiling timing attribute #{attr} was never started." if start_time.nil?
+        t = Time.now.to_f - start_time
+        @time_start[attr]=nil
+        record attr, (Time.now.to_f - start_time)
+      end
+      def record(attr, val)
+        attr = attr.to_sym
+        @times_recorded[attr]||=0
+        @times_recorded[attr]+=1
+        @time[attr]=0 if @time[attr].nil? || @time[attr]=='?'
+        @time[attr]+=val
+      end
+      def to_s
+        mapped = @time.map do |k,v|
+          v = v.round(4) if Numeric === v
+          if (times = @times_recorded[k]) != 1
+            "#{k}(#{times} times):#{v}"
+          else
+            "#{k}:#{v}"
+          end
+        end
+        mapped.join ", "
+      end
+    end
+  end
+end

data/lib/queris/query/trace.rb

@@ -0,0 +1,108 @@
+# encoding: utf-8
+module Queris
+  class Query
+    private
+    class Trace
+      class TraceBase
+        def indent(n=nil)
+          @indent=n if n
+          "  " * indentation
+        end
+        def indentation
+          @indent || 0
+        end
+        def set_options(opt={})
+          @opt=opt
+        end
+        def fval(val) #future value
+          begin
+            Redis::Future === val ? val.value : val
+          rescue Redis::FutureNotReady => e
+            "unavailable"
+          end
+        end
+      end
+      class TraceMessage < TraceBase
+        def initialize(txt)
+          @text=txt
+        end
+        def to_s
+          "#{indent}#{fval @text}"
+        end
+      end
+      class TraceOp < TraceBase #query operation tracer
+        def initialize(redis, operation, operand, results_key)
+          @redis=redis
+          @op=operation
+          @operand=operand
+          @results_key=results_key
+          if Array === @operand.key
+            raise NotImplemented, "Only single-key operands can be traced." 
+          end
+          prepare
+        end
+        
+        def prepare
+          @index_key = @op.operand_key @operand
+          @futures= {
+            :results_size => Queris.run_script(:multisize, @redis, [@results_key]),
+            :results_type => @redis.type(@results_key),
+            :operand_size => Queris.run_script(:multisize, @redis, [@index_key]),
+            :operand_type => @redis.type(@index_key),
+          }
+        end
+        def fval(name)
+          val = @futures[name.to_sym]
+          super val
+        end
+        def to_s
+          op_info = fval(:operand_type) == 'none' ? "key absent" : "|#{fval :operand_type} key|=#{fval :operand_size}"
+          op_key = "#{"[#{@index_key}] " if @opt[:keys]}"
+          if @operand.is_query?
+            "#{indent}#{@op.symbol} subquery<#{@operand.index.id}> #{op_key}(#{op_info}) => #{fval :results_size}\r\n" + 
+            "#{@operand.index.trace(@opt.merge(:output => false, :indent => indentation + 1))}"
+          else
+            "#{indent}#{@op.symbol} #{@operand.index.name}#{@operand.value.nil? ? '' : "<#{@operand.value}>"} #{op_key}(#{op_info}) => #{fval :results_size}"
+          end
+        end
+        
+        private
+        
+      end
+      
+      attr_accessor :buffer
+      def initialize(query, opt={})
+        @query=query
+        @buffer = []
+        @indentation = 0
+        @options= Hash === opt ? opt : {}
+      end
+      def to_s
+        out=[]
+        buffer.each do |line|
+          out << line.to_s
+          out << line.subquery.trace(@indentation + 1) if line.respond_to?(:subquery) && line.subquery
+        end
+        out.join "\r\n"
+      end
+      def indent(n=1)
+        @indentation += n
+        @buffer.each { |line| line.indent n }
+        self
+      end
+      def op(*arg)
+        t = TraceOp.new(@query.redis, *arg)
+        t.set_options @options
+        buffer << t
+        self
+      end
+      def message(text)
+        t = TraceMessage.new(text)
+        t.set_options @options
+        buffer << t
+        self
+      end
+    end
+    
+  end
+end

data/lib/queris/query_store.rb

@@ -0,0 +1,137 @@
+module Queris
+  class QueryStore < Queris::Model
+    index_attribute name: :index, attribute: :all_live_indices, key: :marshaled, value: (proc do |index|
+      case index
+      when Enumerable
+        index.map{|i| QueryStore.index_to_val i}
+      when Index
+        QueryStore.index_to_val index
+      else
+        index
+      end
+    end)
+    index_only
+    live_queries
+    
+    class << self
+      @metaquery_ttl = 600
+      attr_accessor :metaquery_ttl
+      def redis(another_model = nil)
+        another_model = another_model.model if Query === another_model
+        if another_model == self
+          r = Queris.redis "metaquery:metaquery"
+        else
+          r= Queris.redis :'metaquery:slave', :metaquery
+        end
+        raise Error, "No appropriate redis connection found for QueryStore. Add a queris connection with the metaquery role (Queris.add_redis(r, :metaquery), or add live_queries to desired models." unless r
+        r
+      end
+      def redis_master
+        Queris.redis :metaquery, :master
+      end
+      
+      def index_to_val(index)
+        Index === index ? "#{index.model.name}:#{index.class.name.split('::').last}:#{index.name}" : index
+      end
+
+      def add(query)
+        redis.pipelined do
+          redis_indices.each {|i| i.add query}
+          update query
+        end
+        #puts "added #{query} to QueryStore"
+      end
+      def remove(query)
+        redis.pipelined do
+          redis_indices.each { |i| i.remove query }
+        end
+        #puts "removed #{query} from QueryStore"
+      end
+      def update(query)
+        redis.pipelined do
+          redis.setex "Queris:Metaquery:expire:#{query.marshaled}", query.ttl, ""
+        end
+        #puts "updated #{query} for QueryStore"
+      end
+
+      #NOT EFFICIENT!
+      def all_metaqueries
+        q=query(self, :ttl => 20).static!
+        redis_indices(live: true).each { |i| q.union(i) }
+        q.results
+      end
+      
+      def query(model=nil, arg={})
+        model ||= self
+        Metaquery.new(self, arg.merge(:target_model => model, :realtime => true))
+      end
+      def metaquery(arg={})
+        query self, arg
+      end
+      
+      def load(marshaled)
+        Marshal.load(marshaled)
+      end
+      alias :find :load
+      
+      #wipe all metaquery info
+      def clear!
+        querykeys = redis_master.keys query(self).results_key(nil, "*")
+        expirekeys = redis_master.keys "Queris:Metaquery:expire:*"
+        indexkeys = redis_master.keys redis_index(:index).key("*", nil, true)
+        total = querykeys.count + expirekeys.count + indexkeys.count
+        print "Clearing everything (#{total} keys) from QueryStore..."
+        [querykeys, expirekeys, indexkeys].each do |keys|
+          redis_master.multi do |r|
+            keys.each { |k| r.del k }
+          end
+        end
+        puts "ok"
+        total
+      end
+      alias :clear_queries! :clear!
+    end
+    class Metaquery < QuerisModelQuery
+      def initialize(model, arg={})
+        @target_model = arg[:target_model]
+        arg[:profiler] = Queris::DummyProfiler.new
+        super model, arg
+      end
+      def redis_master
+        Queris.redis(model == QueryStore ? :'metaquery:metaquery' : :metaquery)
+      end
+      def redis
+        if model == QueryStore
+          Queris.redis :'metaquery:metaquery'
+        else
+          @redis || Queris::redis(:'metaquery:slave') || redis_master
+        end
+      end
+
+      def results_with_gc
+        res = results(:replace_command => true) do |cmd, key, first, last, rangeopt|
+          redis.evalsha(Queris.script_hash(:results_with_ttl), [key], ["Queris:Metaquery:expire:%s"])
+        end
+        res = [[],[]] if res.empty?
+        res.first.map! do |marshaled|
+          QueryStore.load marshaled
+        end
+        #garbage-collect the expired stuff
+        res.last.each do |marshaled|
+          QueryStore.remove QueryStore.load(marshaled)
+        end
+        res.first
+      end
+      def exists?
+        super(redis)
+      end
+      def set_param_from_index(*arg); self; end
+      %w( union diff intersect ).each do |op|
+        define_method op do |index|
+          index = @target_model.redis_index(index)
+          super model.redis_index(:index), QueryStore.index_to_val(index)
+        end
+      end
+    end
+  end
+end

data/lib/queris/version.rb

@@ -0,0 +1,3 @@
+module Queris
+  VERSION = "0.8.1"
+end

data/lib/rails/log_subscriber.rb

@@ -0,0 +1,22 @@
+module Queris
+  class LogSubscriber < ActiveSupport::LogSubscriber
+    def self.runtime=(value)
+      Thread.current["queris_redis_runtime"] = value
+    end
+
+    def self.runtime
+      Thread.current["queris_redis_runtime"] ||= 0
+    end
+
+    def self.reset_runtime
+      rt, self.runtime = runtime, 0
+      rt
+    end
+
+    def command(event)
+      self.class.runtime += event.duration
+    end
+  end
+end
+
+Queris::LogSubscriber.attach_to :queris

data/lib/rails/request_timing.rb

@@ -0,0 +1,29 @@
+module Queris
+  module ControllerRuntime
+    extend ActiveSupport::Concern
+
+    protected
+
+    def append_info_to_payload(payload)
+      super
+      payload[:redis_queris_runtime] = Queris::LogSubscriber.reset_runtime
+      if Queris.log_stats_per_request?
+        payload[:queris_stats] = Queris::RedisStats.summary
+        Queris::RedisStats.reset
+      end
+    end
+
+    module ClassMethods
+      def log_process_action(payload)
+        messages, queris_runtime = super, payload[:redis_queris_runtime]
+        logger.info "Queris stats by server\r\n" + payload[:queris_stats] if payload[:queris_stats]
+        messages << ("Redis via Queris: %.1fms" % queris_runtime.to_f) if queris_runtime
+        messages
+      end
+    end
+  end
+end
+
+ActiveSupport.on_load(:action_controller) do
+  include Queris::ControllerRuntime
+end

data/lib/tasks/queris.rake

@@ -0,0 +1,138 @@
+#!ruby
+namespace :queris do
+  def confirm
+    if $stdin.respond_to? 'getch'
+      $stdin.getch.upcase == "Y"
+    else #ruby <= 1.9.2 doesn't have getch
+      $stdin.readline[0].upcase == "Y"
+    end
+  end
+  def warn(action=nil, warning=nil, times=1)
+    puts warning if warning
+    if action then
+      q = "Do you #{times>1 ? 'really ' : ''}want to #{action}?"
+    else
+      if times == 1
+        q = "Did you back up everything you needed to back up?"
+      else
+        q = "Are you sure you want to proceed?"
+      end
+    end
+    puts "#{q} [y/n]"
+    if confirm
+      if times <= 1
+        return true
+      else
+        return warn(nil, nil, times-1)
+      end
+    end
+  end
+
+  def build_index(index, check_existence=true, incremental=false)
+    if check_existence
+      model = index.model
+      print "Checking if index #{index.name} already exists..." 
+      foundkeys = index.respond_to?('keypattern') ? model.redis.keys(index.keypattern) : []
+      if foundkeys.count > 0
+        puts "it does."
+        if incremental
+          puts "#{model.name} #{index.name} index data will be deleted incrementally, per element. This runs safely on live data."
+        else
+          puts "All #{model.name} #{index.name} index data will be deleted. Make sure you have a backup!"
+        end
+        return false unless warn
+        if incremental
+          puts "This index spans #{foundkeys.count} redis keys. Every element must be deleted from each of those keys. This may take a while and will slow down redis."
+          if foundkeys.count > 500
+            return false unless warn "continue"
+          end
+        end
+      else
+        puts "it doesn't."
+      end
+    end
+    puts "Building index #{index.name} for #{model.name}"
+    model.build_redis_index index.name, incremental
+  end
+  
+  def load_models
+    # Load all the application's models. Courtesy of random patch for Sunspot ()
+    Rails.application.eager_load! if defined? Rails
+  end
+  desc "Rebuild all queris indices, optionally deleting nearly everything beforehand"
+  task :rebuild, [:clear] => :environment do |t, args|
+    args.with_defaults(:clear => false)
+    abort unless warn "rebuild all redis indices", "All current redis indices and queries will be destroyed!", 3
+    load_models
+    Queris.rebuild!(args.clear)
+  end
+
+  desc "Build all missing indices or a given redis index in the given model"
+  task :'build', [:model, :index, :incremental] => :environment do |t, args|
+    load_models
+    #abort "Please specify a model." if args.model.nil?
+    #abort "Please specify an index." if args.index.nil?
+    if args.model && args.model.length > 0 then
+      begin
+        model = Object.const_get args.model
+        models = [ model ]
+      rescue NameError
+        abort "No model #{args.model} found."
+      end
+    else
+      models = Queris.models
+    end
+
+    if args.index
+      begin 
+        index = model.redis_index args.index
+      rescue
+        abort "No index named #{args.index} found in #{model.name}."
+      end
+    end
+    
+    if model and index then
+      #just one index to build
+      build_index index, true, !!args.incremental
+    else
+      models.each do |model|
+        missing = []
+        model.redis_indices.each do |i|
+          missing << i unless i.skip_create? || i.exists?
+        end
+        if missing.count > 0
+          puts "#{model.name} is missing #{missing.count} #{missing.count  == 1 ? 'index' : 'indices'}: #{missing.map(&:name).join(', ')}."
+          model.build_redis_indices missing
+        else
+          puts "#{model.name} indices already built."
+        end
+      end
+    end
+  end 
+  
+  desc "Clear all object caches"
+  task :clear_cache => :environment do
+    load_models
+    puts "Deleted #{Queris.clear_cache!} cache keys."
+  end
+  
+  desc "Clear all queries"
+  task :clear_queries => :environment do
+    load_models
+    abort unless warn "clear all queries", "All queries, live and otherwise, and all metaqueries will be deleted"
+    puts "Deleted #{Queris.clear_queries!} query keys."
+  end
+  
+  desc "Clear all caches and queries"
+  task :clear => :environment do
+    load_models
+    abort unless warn "clear all queries and caches", "All caches and queries will be deleted", 2
+    puts "Deleted #{Queris.clear!} keys."
+  end
+
+  desc "Queris data sumaries"
+  task :info => :environment do
+    load_models
+    Queris.info
+  end
+end