rediconn 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6ca73adca8e6a1b981b52e05f3ba3eedc3ec1e4043c564128ad47646d9153014
+  data.tar.gz: 8a8582b2e4a53e6719bc49176d43524359bf8bba278c81506edd65cb35cb0204
+SHA512:
+  metadata.gz: 712cb5abbb5efb2eece4ede5624f3d2660bce499c654fa6b2cd8e9780293298f59a3e9e1d130237502c6fb74547f643e2c41dcd8754c6c347da5c57f182e0160
+  data.tar.gz: 906d3a5416e33d164b7705b05512ee43e1932245c493692e3ace4169d091d1a6115aa079e10202166399dc352fb9e0da5763ee363978dfb3db872edb92071f07

data/README.md

@@ -0,0 +1 @@
+# RediConn

data/lib/rediconn/redis_connection.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require "connection_pool"
+require "redis"
+require "uri"
+
+# Adapted from Sidekiq 6.x Implementation
+
+module RediConn
+  class RedisConnection < ConnectionPool
+    KNOWN_REDIS_URL_ENVS = %w[REDIS_URL REDISTOGO_URL REDISCLOUD_URL].freeze
+
+    class << self
+      def configured?(prefix, explicit: true)
+        determine_redis_provider(prefix: prefix, explicit: explicit).present?
+      end
+
+      def shared_pools
+        @shared_pools ||= {}
+      end
+
+      def create(raw_options = {})
+        options = raw_options.transform_keys(&:to_sym)
+
+        if !options[:url] && (u = determine_redis_provider(prefix: raw_options[:env_prefix]))
+          options[:url] = u
+        end
+
+        options[:timeout] ||= 1
+        options[:size] ||= if options[:env_prefix] && (v = ENV["#{options[:env_prefix]}_REDIS_POOL_SIZE"])
+          Integer(v)
+        elsif ENV["RAILS_MAX_THREADS"]
+          Integer(ENV["RAILS_MAX_THREADS"])
+        else
+          5
+        end
+
+        if options[:dedicated]
+          initialize_pool(options)
+        else
+          url = options[:url]
+          cpool = shared_pools[url] ||= initialize_pool(options)
+
+          cpool.resize!(options[:size])
+
+          # TODO Would be nice to respect timeouts
+
+          cpool
+        end
+      end
+
+      private
+
+      def client_opts(options)
+        opts = options.dup
+        if opts[:namespace]
+          opts.delete(:namespace)
+        end
+
+        opts.delete(:size)
+        opts.delete(:env_prefix)
+
+        if opts[:network_timeout]
+          opts[:timeout] = opts[:network_timeout]
+          opts.delete(:network_timeout)
+        end
+
+        opts[:reconnect_attempts] ||= 1
+
+        opts
+      end
+
+      def determine_redis_provider(prefix: nil, explicit: false)
+        vars = []
+
+        if prefix.present?
+          if (ptr = ENV["#{prefix}_REDIS_PROVIDER"]).present?
+            return ENV[ptr]
+          else
+            vars.push(*KNOWN_REDIS_URL_ENVS.map { |e| ENV["#{prefix}_#{e}"] })
+          end
+        end
+
+        if !explicit || !prefix.present?
+          if (ptr = ENV["REDIS_PROVIDER"]).present?
+            vars << ptr # Intentionally not a return
+          else
+            vars.push(*KNOWN_REDIS_URL_ENVS)
+          end
+        end
+
+        vars.select!(&:present?)
+
+        vars.each do |e|
+          return ENV[e] if ENV[e].present?
+        end
+
+        nil
+      end
+
+      def initialize_pool(options)
+        new(timeout: options[:timeout], size: options[:size]) do
+          namespace = options[:namespace]
+          client = Redis.new client_opts(options)
+
+          if namespace
+            begin
+              require "redis/namespace"
+              Redis::Namespace.new(namespace, redis: client)
+            rescue LoadError
+              Rails.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
+                                   "Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
+              exit(-127)
+            end
+          else
+            client
+          end
+        end
+      end
+    end
+
+    # Mainly intended for console use
+    def lazy
+      RedisProxy.new do |&blk|
+        with(&blk)
+      end
+    end
+
+    def lazy_with(&blk)
+      return lazy unless blk
+      with(&blk)
+    end
+
+    # Very naughty, but looking at the source, should be safe
+    def resize!(new_size)
+      return unless new_size > @size
+      @size = new_size
+      @available.instance_variable_set(:@max, new_size)
+    end
+  end
+end

data/lib/rediconn/redis_model.rb

@@ -0,0 +1,67 @@
+module RediConn
+  module RedisModel
+    extend ActiveSupport::Concern
+
+    class_methods do
+      def redis_attr(key, type = :string, read_only: true)
+        class_eval <<-RUBY, __FILE__, __LINE__ + 1
+          def #{key}=(value)
+            raise "#{key} is read-only once the batch has been started" if #{read_only.to_s} && (@initialized || @existing)
+            @#{key} = value
+            if :#{type} == :json
+              value = JSON.unparse(value)
+            end
+            persist_bid_attr('#{key}', value)
+          end
+
+          def #{key}
+            return @#{key} if defined?(@#{key})
+            if (@initialized || @existing)
+              value = read_bid_attr('#{key}')
+              if :#{type} == :bool
+                value = value == 'true'
+              elsif :#{type} == :int
+                value = value.to_i
+              elsif :#{type} == :float
+                value = value.to_f
+              elsif :#{type} == :json
+                value = JSON.parse(value)
+              elsif :#{type} == :symbol
+                value = value&.to_sym
+              end
+              @#{key} = value
+            end
+          end
+        RUBY
+      end
+    end
+
+    def persist_bid_attr(attribute, value)
+      if @initialized || @existing
+        redis do |r|
+          r.multi do |r|
+            r.hset(redis_key, attribute, value.to_s)
+            r.expire(redis_key, Batch::BID_EXPIRE_TTL)
+          end
+        end
+      else
+        @pending_attrs ||= {}
+        @pending_attrs[attribute] = value.to_s
+      end
+    end
+
+    def read_bid_attr(attribute)
+      redis do |r|
+        r.hget(redis_key, attribute)
+      end
+    end
+
+    def flush_pending_attrs
+      redis do |r|
+        r.mapped_hmset(redis_key, @pending_attrs)
+      end
+      @initialized = true
+      @pending_attrs = {}
+    end
+  end
+end

data/lib/rediconn/redis_proxy.rb

@@ -0,0 +1,51 @@
+module RediConn
+  # Mainly intended for console use
+  class RedisProxy
+    def initialize(&factory)
+      @factory = factory
+    end
+
+    def multi(*args, &block)
+      @factory.call do |r|
+        r.multi(*args) do |r|
+          block.call(r)
+        end
+      end
+    end
+
+    def pipelined(*args, &block)
+      @factory.call do |r|
+        r.pipelined(*args) do |r2|
+          block.call(r2 || r)
+        end
+      end
+    end
+
+    def uget(key)
+      @factory.call do |r|
+        case r.type(key)
+        when 'string'
+          r.get(key)
+        when 'list'
+          r.lrange(key, 0, -1)
+        when 'hash'
+          r.hgetall(key)
+        when 'set'
+          r.smembers(key)
+        when 'zset'
+          r.zrange(key, 0, -1)
+        end
+      end
+    end
+
+    def method_missing(method_name, *arguments, &block)
+      @factory.call do |r|
+        r.send(method_name, *arguments, &block)
+      end
+    end
+
+    def respond_to_missing?(method_name, include_private = false)
+      super || Redis.method_defined?(method_name)
+    end
+  end
+end

data/lib/rediconn/redis_script.rb

@@ -0,0 +1,161 @@
+require 'pathname'
+require 'digest/sha1'
+require 'erb'
+
+# Modified from https://github.com/Shopify/wolverine/blob/master/lib/wolverine/script.rb
+
+module RediConn
+  # {RedisScript} represents a lua script in the filesystem. It loads the script
+  # from disk and handles talking to redis to execute it. Error handling
+  # is handled by {LuaError}.
+  class RedisScript
+
+    # Loads the script file from disk and calculates its +SHA1+ sum.
+    #
+    # @param file [Pathname] the full path to the indicated file
+    def initialize(file)
+      @file = Pathname.new(file)
+    end
+
+    # Passes the script and supplied arguments to redis for evaulation.
+    # It first attempts to use a script redis has already cached by using
+    # the +EVALSHA+ command, but falls back to providing the full script
+    # text via +EVAL+ if redis has not seen this script before. Future
+    # invocations will then use +EVALSHA+ without erroring.
+    #
+    # @param redis [Redis] the redis connection to run against
+    # @param args [*Objects] the arguments to the script
+    # @return [Object] the value passed back by redis after script execution
+    # @raise [LuaError] if the script failed to compile of encountered a
+    #   runtime error
+    def call(redis, *args)
+      t = Time.now
+      begin
+        redis.evalsha(digest, *args)
+      rescue => e
+        e.message =~ /NOSCRIPT/ ? redis.eval(content, *args) : raise
+      end
+    rescue => e
+      if LuaError.intercepts?(e)
+        raise LuaError.new(e, @file, content)
+      else
+        raise
+      end
+    end
+
+    def content
+      @content ||= load_lua(@file)
+    end
+
+    def digest
+      @digest ||= Digest::SHA1.hexdigest content
+    end
+
+    private
+
+    def script_path
+      Rails.root + 'app/redis_lua'
+    end
+
+    def relative_path
+      @path ||= @file.relative_path_from(script_path)
+    end
+
+    def load_lua(file)
+      TemplateContext.new(script_path).template(script_path + file)
+    end
+
+    class TemplateContext
+      def initialize(script_path)
+        @script_path = script_path
+      end
+
+      def template(pathname)
+        @partial_templates ||= {}
+        ERB.new(File.read(pathname)).result binding
+      end
+
+      # helper method to include a lua partial within another lua script
+      #
+      # @param relative_path [String] the relative path to the script from
+      #     `script_path`
+      def include_partial(relative_path)
+        unless @partial_templates.has_key? relative_path
+          @partial_templates[relative_path] = nil
+          template( Pathname.new("#{@script_path}/#{relative_path}") )
+        end
+      end
+    end
+
+    # Reformats errors raised by redis representing failures while executing
+    # a lua script. The default errors have confusing messages and backtraces,
+    # and a type of +RuntimeError+. This class improves the message and
+    # modifies the backtrace to include the lua script itself in a reasonable
+    # way.
+    class LuaError < StandardError
+      PATTERN = /ERR Error (compiling|running) script \(.*?\): .*?:(\d+): (.*)/
+      WOLVERINE_LIB_PATH = File.expand_path('../../', __FILE__)
+      CONTEXT_LINE_NUMBER = 2
+
+      attr_reader :error, :file, :content
+
+      # Is this error one that should be reformatted?
+      #
+      # @param error [StandardError] the original error raised by redis
+      # @return [Boolean] is this an error that should be reformatted?
+      def self.intercepts? error
+        error.message =~ PATTERN
+      end
+
+      # Initialize a new {LuaError} from an existing redis error, adjusting
+      # the message and backtrace in the process.
+      #
+      # @param error [StandardError] the original error raised by redis
+      # @param file [Pathname] full path to the lua file the error ocurred in
+      # @param content [String] lua file content the error ocurred in
+      def initialize error, file, content
+        @error = error
+        @file = file
+        @content = content
+
+        @error.message =~ PATTERN
+        _stage, line_number, message = $1, $2, $3
+        error_context = generate_error_context(content, line_number.to_i)
+
+        super "#{message}\n\n#{error_context}\n\n"
+        set_backtrace generate_backtrace file, line_number
+      end
+
+      private
+
+      def generate_error_context(content, line_number)
+        lines = content.lines.to_a
+        beginning_line_number = [1, line_number - CONTEXT_LINE_NUMBER].max
+        ending_line_number = [lines.count, line_number + CONTEXT_LINE_NUMBER].min
+        line_number_width = ending_line_number.to_s.length
+
+        (beginning_line_number..ending_line_number).map do |number|
+          indicator = number == line_number ? '=>' : '  '
+          formatted_number = "%#{line_number_width}d" % number
+          " #{indicator} #{formatted_number}: #{lines[number - 1]}"
+        end.join.chomp
+      end
+
+      def generate_backtrace(file, line_number)
+        pre_wolverine = backtrace_before_entering_wolverine(@error.backtrace)
+        index_of_first_wolverine_line = (@error.backtrace.size - pre_wolverine.size - 1)
+        pre_wolverine.unshift(@error.backtrace[index_of_first_wolverine_line])
+        pre_wolverine.unshift("#{file}:#{line_number}")
+        pre_wolverine
+      end
+
+      def backtrace_before_entering_wolverine(backtrace)
+        backtrace.reverse.take_while { |line| ! line_from_wolverine(line) }.reverse
+      end
+
+      def line_from_wolverine(line)
+        line.split(':').first.include?(WOLVERINE_LIB_PATH)
+      end
+    end
+  end
+end

data/lib/rediconn/version.rb

@@ -0,0 +1,3 @@
+module RediConn
+  VERSION = "0.1.0".freeze
+end

data/lib/rediconn.rb

@@ -0,0 +1,8 @@
+
+require 'active_support/concern'
+
+Dir[File.dirname(__FILE__) + "/rediconn/**/*.rb"].each { |file| require file }
+
+module RediConn
+
+end

data/rediconn.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+begin
+  require "rediconn/version"
+  version = RediConn::VERSION
+rescue LoadError
+  version = "0.0.0.docker"
+end
+
+Gem::Specification.new do |spec|
+  spec.name          = "rediconn"
+  spec.version       = version
+  spec.authors       = ["Ethan Knapp"]
+  spec.email         = ["eknapp@instructure.com"]
+
+  spec.summary       = "Gem for consistent, de-duplicated Redis Connection Pooling"
+  spec.homepage      = "https://instructure.com"
+
+  spec.files = Dir["{app,config,db,lib}/**/*", "README.md", "*.gemspec"]
+  spec.test_files = Dir["spec/**/*"]
+  spec.require_paths = ['lib']
+
+  spec.add_dependency "rails", ">= 5.2", "< 9.0"
+  spec.add_dependency "connection_pool", ">= 2.2.0", "< 3.0"
+
+  spec.add_development_dependency "redis"
+  spec.add_development_dependency 'rspec', '~> 3'
+end

data/spec/rediconn/redis_connection_spec.rb

@@ -0,0 +1,38 @@
+require 'spec_helper'
+
+RSpec.describe RediConn::RedisConnection do
+  let(:conn) { described_class.create(env_prefix: "REDICONN") }
+
+  describe '.redis' do
+    it 'returns the same connection if called when nested' do
+      conn.lazy_with do |r1|
+        conn.lazy_with do |r2|
+          expect(r1).to be r2
+        end
+      end
+    end
+
+    it 'returns a RedisProxy object if no block is given' do
+      expect(conn.lazy).to be_a(RediConn::RedisProxy)
+    end
+
+    describe 'RedisProxy' do
+      let!(:proxy) { conn.lazy }
+
+      it 'calls Batch.redis with a block' do
+        expect(conn).to receive(:with) do |&block|
+          expect(block).to be_a Proc
+        end
+        proxy.sadd('KEY', 5)
+      end
+
+      describe '#multi' do
+        it 'works with 1 arity' do
+          proxy.multi do |r|
+            r.sadd('KEY', 5)
+          end
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+require 'bundler'
+
+Bundler.require :default, :development
+
+require 'active_support'
+require 'active_support/core_ext'
+
+require 'redis'
+
+RSpec.configure do |config|
+
+end

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification
+name: rediconn
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ethan Knapp
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+email:
+- eknapp@instructure.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/rediconn.rb
+- lib/rediconn/redis_connection.rb
+- lib/rediconn/redis_model.rb
+- lib/rediconn/redis_proxy.rb
+- lib/rediconn/redis_script.rb
+- lib/rediconn/version.rb
+- rediconn.gemspec
+- spec/rediconn/redis_connection_spec.rb
+- spec/spec_helper.rb
+homepage: https://instructure.com
+licenses: []
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Gem for consistent, de-duplicated Redis Connection Pooling
+test_files:
+- spec/rediconn/redis_connection_spec.rb
+- spec/spec_helper.rb