rb-cluster-mm 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 297f654807597a7363bcbd0de01ce2ebb2e64c67
+  data.tar.gz: 996a10d446ecde7c892b7a5a43325487b7399d5d
+SHA512:
+  metadata.gz: 8d4dfd14942c627ca5c78ff78626cf8edd7f0bb311f0126c45b6371668f26c2e3870a39fa2e02dc14b76e7d92edd7cfa767e88061081a3ce2734b5eccaf8a1ab
+  data.tar.gz: b575a4b90634b14f10b41a4730fff5efcc2d1d49cc91d6d666e32503665b8411586054ea02ae6817c266c6eb45c03e60b8f5a098e11821d969c1a3bb7013193e

data/lib/crc16.rb

@@ -0,0 +1,85 @@
+# Copyright (C) 2013 Salvatore Sanfilippo <antirez@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+# 
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+# -----------------------------------------------------------------------------
+#
+# This is the CRC16 algorithm used by Redis Cluster to hash keys.
+# Implementation according to CCITT standards.
+#
+# This is actually the XMODEM CRC 16 algorithm, using the
+# following parameters:
+#
+# Name                       : "XMODEM", also known as "ZMODEM", "CRC-16/ACORN"
+# Width                      : 16 bit
+# Poly                       : 1021 (That is actually x^16 + x^12 + x^5 + 1)
+# Initialization             : 0000
+# Reflect Input byte         : False
+# Reflect Output CRC         : False
+# Xor constant to output CRC : 0000
+# Output for "123456789"     : 31C3
+
+module RedisClusterCRC16
+
+    def RedisClusterCRC16.crc16(bytes)
+        crc = 0
+        bytes.each_byte{|b|
+            crc = ((crc<<8) & 0xffff) ^ XMODEMCRC16Lookup[((crc>>8)^b) & 0xff]
+        }
+        crc
+    end
+
+private
+
+    XMODEMCRC16Lookup = [
+        0x0000,0x1021,0x2042,0x3063,0x4084,0x50a5,0x60c6,0x70e7,
+        0x8108,0x9129,0xa14a,0xb16b,0xc18c,0xd1ad,0xe1ce,0xf1ef,
+        0x1231,0x0210,0x3273,0x2252,0x52b5,0x4294,0x72f7,0x62d6,
+        0x9339,0x8318,0xb37b,0xa35a,0xd3bd,0xc39c,0xf3ff,0xe3de,
+        0x2462,0x3443,0x0420,0x1401,0x64e6,0x74c7,0x44a4,0x5485,
+        0xa56a,0xb54b,0x8528,0x9509,0xe5ee,0xf5cf,0xc5ac,0xd58d,
+        0x3653,0x2672,0x1611,0x0630,0x76d7,0x66f6,0x5695,0x46b4,
+        0xb75b,0xa77a,0x9719,0x8738,0xf7df,0xe7fe,0xd79d,0xc7bc,
+        0x48c4,0x58e5,0x6886,0x78a7,0x0840,0x1861,0x2802,0x3823,
+        0xc9cc,0xd9ed,0xe98e,0xf9af,0x8948,0x9969,0xa90a,0xb92b,
+        0x5af5,0x4ad4,0x7ab7,0x6a96,0x1a71,0x0a50,0x3a33,0x2a12,
+        0xdbfd,0xcbdc,0xfbbf,0xeb9e,0x9b79,0x8b58,0xbb3b,0xab1a,
+        0x6ca6,0x7c87,0x4ce4,0x5cc5,0x2c22,0x3c03,0x0c60,0x1c41,
+        0xedae,0xfd8f,0xcdec,0xddcd,0xad2a,0xbd0b,0x8d68,0x9d49,
+        0x7e97,0x6eb6,0x5ed5,0x4ef4,0x3e13,0x2e32,0x1e51,0x0e70,
+        0xff9f,0xefbe,0xdfdd,0xcffc,0xbf1b,0xaf3a,0x9f59,0x8f78,
+        0x9188,0x81a9,0xb1ca,0xa1eb,0xd10c,0xc12d,0xf14e,0xe16f,
+        0x1080,0x00a1,0x30c2,0x20e3,0x5004,0x4025,0x7046,0x6067,
+        0x83b9,0x9398,0xa3fb,0xb3da,0xc33d,0xd31c,0xe37f,0xf35e,
+        0x02b1,0x1290,0x22f3,0x32d2,0x4235,0x5214,0x6277,0x7256,
+        0xb5ea,0xa5cb,0x95a8,0x8589,0xf56e,0xe54f,0xd52c,0xc50d,
+        0x34e2,0x24c3,0x14a0,0x0481,0x7466,0x6447,0x5424,0x4405,
+        0xa7db,0xb7fa,0x8799,0x97b8,0xe75f,0xf77e,0xc71d,0xd73c,
+        0x26d3,0x36f2,0x0691,0x16b0,0x6657,0x7676,0x4615,0x5634,
+        0xd94c,0xc96d,0xf90e,0xe92f,0x99c8,0x89e9,0xb98a,0xa9ab,
+        0x5844,0x4865,0x7806,0x6827,0x18c0,0x08e1,0x3882,0x28a3,
+        0xcb7d,0xdb5c,0xeb3f,0xfb1e,0x8bf9,0x9bd8,0xabbb,0xbb9a,
+        0x4a75,0x5a54,0x6a37,0x7a16,0x0af1,0x1ad0,0x2ab3,0x3a92,
+        0xfd2e,0xed0f,0xdd6c,0xcd4d,0xbdaa,0xad8b,0x9de8,0x8dc9,
+        0x7c26,0x6c07,0x5c64,0x4c45,0x3ca2,0x2c83,0x1ce0,0x0cc1,
+        0xef1f,0xff3e,0xcf5d,0xdf7c,0xaf9b,0xbfba,0x8fd9,0x9ff8,
+        0x6e17,0x7e36,0x4e55,0x5e74,0x2e93,0x3eb2,0x0ed1,0x1ef0
+    ]
+end

data/lib/rediscluster.rb

@@ -0,0 +1,278 @@
+# Copyright (C) 2013 Salvatore Sanfilippo <antirez@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+# 
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'rubygems'
+require 'redis'
+require File.expand_path('../../../support/redis_library/crc16.rb', __FILE__)
+
+class RedisCluster
+
+    RedisClusterHashSlots = 16384
+    RedisClusterRequestTTL = 16
+    RedisClusterDefaultTimeout = 1
+
+    def initialize(startup_nodes,connections,opt={})
+        @startup_nodes = startup_nodes
+        @max_connections = connections
+        @connections = {}
+        @opt = opt
+        @refresh_table_asap = false
+        initialize_slots_cache
+    end
+
+    def get_redis_link(host,port)
+        timeout = @opt[:timeout] or RedisClusterDefaultTimeout
+        Redis.new(:host => host, :port => port, :timeout => timeout)
+    end
+
+    # Given a node (that is just a Ruby hash) give it a name just
+    # concatenating the host and port. We use the node name as a key
+    # to cache connections to that node.
+    def set_node_name!(n)
+        if !n[:name]
+            n[:name] = "#{n[:host]}:#{n[:port]}"
+        end
+    end
+
+    # Contact the startup nodes and try to fetch the hash slots -> instances
+    # map in order to initialize the @slots hash.
+    def initialize_slots_cache
+        @startup_nodes.each{|n|
+            begin
+                @slots = {}
+                @nodes = []
+
+                r = get_redis_link(n[:host],n[:port])
+                r.cluster("slots").each {|r|
+                    (r[0]..r[1]).each{|slot|
+                        ip,port = r[2]
+                        name = "#{ip}:#{port}"
+                        node = {
+                            :host => ip, :port => port,
+                            :name => name
+                        }
+                        @nodes << node
+                        @slots[slot] = node
+                    }
+                }
+                populate_startup_nodes
+                @refresh_table_asap = false
+            rescue
+                # Try with the next node on error.
+                next
+            end
+            # Exit the loop as long as the first node replies
+            break
+        }
+    end
+
+    # Use @nodes to populate @startup_nodes, so that we have more chances
+    # if a subset of the cluster fails.
+    def populate_startup_nodes
+        # Make sure every node has already a name, so that later the
+        # Array uniq! method will work reliably.
+        @startup_nodes.each{|n| set_node_name! n}
+        @nodes.each{|n| @startup_nodes << n}
+        @startup_nodes.uniq!
+    end
+
+    # Flush the cache, mostly useful for debugging when we want to force
+    # redirection.
+    def flush_slots_cache
+        @slots = {}
+    end
+
+    # Return the hash slot from the key.
+    def keyslot(key)
+        # Only hash what is inside {...} if there is such a pattern in the key.
+        # Note that the specification requires the content that is between
+        # the first { and the first } after the first {. If we found {} without
+        # nothing in the middle, the whole key is hashed as usually.
+        s = key.index "{"
+        if s
+            e = key.index "}",s+1
+            if e && e != s+1
+                key = key[s+1..e-1]
+            end
+        end
+        RedisClusterCRC16.crc16(key) % RedisClusterHashSlots
+    end
+
+    # Return the first key in the command arguments.
+    #
+    # Currently we just return argv[1], that is, the first argument
+    # after the command name.
+    #
+    # This is indeed the key for most commands, and when it is not true
+    # the cluster redirection will point us to the right node anyway.
+    #
+    # For commands we want to explicitly bad as they don't make sense
+    # in the context of cluster, nil is returned.
+    def get_key_from_command(argv)
+        case argv[0].to_s.downcase
+        when "info","multi","exec","slaveof","config","shutdown"
+            return nil
+        else
+            # Unknown commands, and all the commands having the key
+            # as first argument are handled here:
+            # set, get, ...
+            return argv[1]
+        end
+    end
+
+    # If the current number of connections is already the maximum number
+    # allowed, close a random connection. This should be called every time
+    # we cache a new connection in the @connections hash.
+    def close_existing_connection
+        while @connections.length >= @max_connections
+            @connections.each{|n,r|
+                @connections.delete(n)
+                begin
+                    r.client.disconnect
+                rescue
+                end
+                break
+            }
+        end
+    end
+
+    # Return a link to a random node, or raise an error if no node can be
+    # contacted. This function is only called when we can't reach the node
+    # associated with a given hash slot, or when we don't know the right
+    # mapping.
+    #
+    # The function will try to get a successful reply to the PING command,
+    # otherwise the next node is tried.
+    def get_random_connection
+        e = ""
+        @startup_nodes.shuffle.each{|n|
+            begin
+                set_node_name!(n)
+                conn = @connections[n[:name]]
+
+                if !conn
+                    # Connect the node if it is not connected
+                    conn = get_redis_link(n[:host],n[:port])
+                    if conn.ping == "PONG"
+                        close_existing_connection
+                        @connections[n[:name]] = conn
+                        return conn
+                    else
+                        # If the connection is not good close it ASAP in order
+                        # to avoid waiting for the GC finalizer. File
+                        # descriptors are a rare resource.
+                        conn.client.disconnect
+                    end
+                else
+                    # The node was already connected, test the connection.
+                    return conn if conn.ping == "PONG"
+                end
+            rescue => e
+                # Just try with the next node.
+            end
+        }
+        raise "Can't reach a single startup node. #{e}"
+    end
+
+    # Given a slot return the link (Redis instance) to the mapped node.
+    # Make sure to create a connection with the node if we don't have
+    # one.
+    def get_connection_by_slot(slot)
+        node = @slots[slot]
+        # If we don't know what the mapping is, return a random node.
+        return get_random_connection if !node
+        set_node_name!(node)
+        if not @connections[node[:name]]
+            begin
+                close_existing_connection
+                @connections[node[:name]] =
+                    get_redis_link(node[:host],node[:port])
+            rescue
+                # This will probably never happen with recent redis-rb
+                # versions because the connection is enstablished in a lazy
+                # way only when a command is called. However it is wise to
+                # handle an instance creation error of some kind.
+                return get_random_connection
+            end
+        end
+        @connections[node[:name]]
+    end
+
+    # Dispatch commands.
+    def send_cluster_command(argv)
+        initialize_slots_cache if @refresh_table_asap
+        ttl = RedisClusterRequestTTL; # Max number of redirections
+        e = ""
+        asking = false
+        try_random_node = false
+        while ttl > 0
+            ttl -= 1
+            key = get_key_from_command(argv)
+            raise "No way to dispatch this command to Redis Cluster." if !key
+            slot = keyslot(key)
+            if try_random_node
+                r = get_random_connection
+                try_random_node = false
+            else
+                r = get_connection_by_slot(slot)
+            end
+            begin
+                # TODO: use pipelining to send asking and save a rtt.
+                r.asking if asking
+                asking = false
+                return r.send(argv[0].to_sym,*argv[1..-1])
+            rescue Errno::ECONNREFUSED, Redis::TimeoutError, Redis::CannotConnectError, Errno::EACCES
+                try_random_node = true
+                sleep(0.1) if ttl < RedisClusterRequestTTL/2
+            rescue => e
+                errv = e.to_s.split
+                if errv[0] == "MOVED" || errv[0] == "ASK"
+                    if errv[0] == "ASK"
+                        asking = true
+                    else
+                        # Serve replied with MOVED. It's better for us to
+                        # ask for CLUSTER NODES the next time.
+                        @refresh_table_asap = true
+                    end
+                    newslot = errv[1].to_i
+                    node_ip,node_port = errv[2].split(":")
+                    if !asking
+                        @slots[newslot] = {:host => node_ip,
+                                           :port => node_port.to_i}
+                    end
+                else
+                    raise e
+                end
+            end
+        end
+        raise "Too many Cluster redirections? (last error: #{e})"
+    end
+
+    # Currently we handle all the commands using method_missing for
+    # simplicity. For a Cluster client actually it will be better to have
+    # every single command as a method with the right arity and possibly
+    # additional checks (example: RPOPLPUSH with same src/dst key, SORT
+    # without GET or BY, and so forth).
+    def method_missing(*argv)
+        send_cluster_command(argv)
+    end
+end
+

metadata

@@ -0,0 +1,45 @@
+--- !ruby/object:Gem::Specification
+name: rb-cluster-mm
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- ysh1986
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-06-27 00:00:00.000000000 Z
+dependencies: []
+description: redis cluster connect
+email: ysh198606@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/crc16.rb
+- lib/rediscluster.rb
+homepage: https://rubygems.org/profiles/ysh1986
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: rb-cluster-mm
+test_files: []