migrate-ssdb2pika 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 14d100832a2fcc8b020eda9a27aa38cd55859f077b69753a56242b7a39af13c5
+  data.tar.gz: a8053aecb1241cde07d94afa5785eeabd0026e76e206d519aaa6f90ecbec39bf
+SHA512:
+  metadata.gz: bd9db88bf123cdeca2f46ccd60344169a4aec4e4981590a207b4d1b8dcb374917dcd15bc17f0f1702bbf96298910492089c1fd2a39236cc2fc158f2dcbf45152
+  data.tar.gz: 931ae8685de07b2c3d0673cbdd985984f94cfadca8551948de417d1fbab2dd49c83316a347c247ba3b1490f5e17474ee119be6e4afe59aed02df4c5aa00b4a8b

data/.gitignore

@@ -0,0 +1,2 @@
+
+/config/redis.yml

data/Gemfile

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+gemspec
+
+gem "ssdb",
+  git: "https://github.com/xiewenwei/ssdb-rb.git",
+  branch: 'feature/support-hlist-and-hscan'

data/Gemfile.lock

@@ -0,0 +1,31 @@
+GIT
+  remote: https://github.com/xiewenwei/ssdb-rb.git
+  revision: 911a14a06545431b547fb61379050c3a53b0ce29
+  branch: feature/support-hlist-and-hscan
+  specs:
+    ssdb (0.1.3)
+
+PATH
+  remote: .
+  specs:
+    migrate-ssdb2pika (0.1.0)
+      redis (>= 3)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    minitest (5.11.3)
+    rake (10.5.0)
+    redis (4.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  migrate-ssdb2pika!
+  minitest (~> 5.0)
+  rake (~> 10.0)
+  ssdb!
+
+BUNDLED WITH
+   1.16.2

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2019 Vincent Xie
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,86 @@
+
+# 迁移 SSDB 到 Redis/Pika 工具
+
+该 Gem 用于 迁移 SSDB 到 Redis/Pika, 包括支持双写工具和迁移历史数据工具，下面分别介绍用法。
+
+## 1、SSDB 和 Redis/Pika 双写代理
+
+### 使用方法
+
+* 在目标项目（通常是 Rails 应用）引入 `migrate-ssdb2pika` gem
+
+```ruby
+  gem 'migrate-ssdb2pika',
+    git: 'https://github.com/xiewenwei/migrate-ssdb2pika.git'
+```
+
+* 配置 SSDB 连接生成以支持双写
+
+例如：
+
+```ruby
+  ssdb = Redis.new(host: '192.168.1.3', port: 7981)
+  pika = Redis.new(host: '192.168.1.5', port: 7981)
+
+  $ssdb = MigrateSsdb2pika.new_client(master: ssdb, slave: pika, master_is_ssdb: true)
+```
+
+$ssdb 就是支持双写的 client connection
+
+### 特别注意事项
+
+SSDB 使用 zclear 方法删除 zset 的 key，而 Redis/Pika 并不支持 zclear 方法，所以需要特殊处理。
+
+如果原来有直接使用 zclear 的话，需要改写成如下的形式：
+
+```ruby
+  if $ssdb.respond_to? :clear_zset
+    $ssdb.clear_zset key
+  else
+    $ssdb.del key
+  end
+```
+
+迁移完成后改为只使用 `$ssdb.del key` 即可。
+
+SSDB 和 Redis/Pika 双写代理和迁移历史数据工具尽管放在一个 Gem 里，但它们是相互独立的。
+
+## 2、迁移历史数据工具
+
+### 安装
+
+* 克隆 migrate-ssdb2pika 项目到运行机器
+* 执行 `bundle install`
+
+### 运行方法
+
+```shell
+bin/ssdb2pika --ssdb_host=xxx --ssdb_port=xxx \
+  --pika_host=xxx --pika_port=xx \
+  -m <kv|hash|zset|queue|all>
+```
+
+例如：
+
+```shell
+bin/ssdb2pika --ssdb_host=192.168.0.10 --ssdb_port=7981 \
+--pika_host=192.168.0.12 --pika_port=7981 -m all
+```
+
+如果担心执行时间过长，可以使用 `nohup` 方式执行。
+
+```shell
+nohup bin/ssdb2pika --ssdb_host=192.168.0.10 --ssdb_port=7981 \
+--pika_host=192.168.0.12 --pika_port=7981 -m all &
+```
+
+### 原理
+
+直接使用 `ssdb-rb` sdk
+
+* 通过 `scan` 遍历 key-value 数据
+* 通过 `hlist` 遍历 hash 所有 name，通过 `hscan` 遍历某一个 hash 里所有 field-value
+* 通过 `zlist` 遍历 zset 所有 name，通过 `zscan` 遍历某一个 zset 里所有 member-score
+* 通过 `qlist` 遍历 queue 所有 name
+
+更改 `ssdb-rb` 代码以支持 hlist/hscan/qlist/qrange

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task default: :test

data/bin/ssdb2pika

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require "rubygems"
+require "bundler/setup"
+
+require 'optparse'
+require 'migrate_ssdb2pika/migration'
+
+def parse_options
+  options = {
+    mode: 'all',
+    write: 'yes'
+  }
+
+  optparse = OptionParser.new do |opts|
+    opts.banner = "Usage: bin/ssdb2pika <options>"
+
+    opts.on('', '--ssdb_host host', 'host of ssdb') do |value|
+      options[:ssdb_host] = value
+    end
+    opts.on('', '--ssdb_port port', 'port of ssdb') do |value|
+      options[:ssdb_port] = value
+    end
+
+    opts.on('', '--pika_host host', 'host of pika/redis') do |value|
+      options[:pika_host] = value
+    end
+    opts.on('', '--pika_port port', 'port of pika/redis') do |value|
+      options[:pika_port] = value
+    end
+
+    opts.on('-m', '--mode mode', 'Mode (kv/hash/zset/queue default all)') do |value|
+      options[:mode] = value
+    end
+    opts.on('-w', '--write write', 'write pika or not (yes/no)') do |value|
+      options[:write] = value
+    end
+  end
+
+  # 检查参数
+  begin
+    optparse.parse!
+    mandatory = [:ssdb_host, :ssdb_port, :pika_host, :pika_port]
+    missing = mandatory.select{ |param| options[param].nil? }
+    raise OptionParser::MissingArgument, missing.join(', ') unless missing.empty?
+    options
+  rescue OptionParser::ParseError => e
+    puts e
+    puts optparse
+    exit
+  end
+
+end
+
+MigrateSsdb2pika::Migration.new(parse_options).start

data/lib/migrate-ssdb2pika.rb

@@ -0,0 +1,9 @@
+
+require 'migrate_ssdb2pika/version'
+require 'migrate_ssdb2pika/redis_ssdb_proxy'
+
+module MigrateSsdb2pika
+  def self.new_client(*args)
+    RedisSsdbProxy.new(*args)
+  end
+end # MigrateSsdb2pika

data/lib/migrate_ssdb2pika/migration.rb

@@ -0,0 +1,289 @@
+
+require 'redis'
+require 'ssdb'
+
+module MigrateSsdb2pika
+  class Migration
+    SCAN_LIMIT = 100
+
+    def initialize(options)
+      @options = options
+    end
+
+    def start
+      setup_client
+      log_message("start")
+      case @options[:mode]
+        when 'kv'
+          scan_key_values
+          log_message("scan_key_values done")
+        when 'hash'
+          scan_all_hashs
+          log_message("scan_all_hashs done")
+        when 'zset'
+          scan_all_zsets
+          log_message("scan_all_zsets done")
+        when 'queue'
+          scan_all_queues
+          log_message("scan_all_queues done")
+        else
+          scan_key_values
+          log_message("scan_key_values done")
+          scan_all_hashs
+          log_message("scan_all_hashs done")
+          scan_all_zsets
+          log_message("scan_all_zsets done")
+          scan_all_queues
+          log_message("scan_all_queues done")
+      end
+    end
+
+    private
+
+    def log_message(msg)
+      puts "[#{Time.now.to_s}] #{msg}"
+    end
+
+    def setup_client
+      @ssdb = SSDB.new url: "ssdb://#{@options[:ssdb_host]}:#{@options[:ssdb_port]}"
+      @ssdb_redis = Redis.new url: "redis://#{@options[:ssdb_host]}:#{@options[:ssdb_port]}"
+      @pika = Redis.new url: "redis://#{@options[:pika_host]}:#{@options[:pika_port]}"
+    end
+
+    # key-value scan
+    def scan_key_values
+      count = 0
+      key_start = ''
+      while true
+        entries = @ssdb.scan(key_start, '', limit: SCAN_LIMIT)
+        count += 1
+        puts "scan_key_values #{count} name_start: #{key_start}"
+        if entries.size == 0
+          break
+        end
+
+        array = []
+        entries.each do |key, value|
+          array << key
+          array << value
+        end
+
+        if @options[:write] == 'yes'
+          @pika.mset *array
+        else
+          puts "array size #{array.size}"
+        end
+        key_start = entries.last[0]
+      end
+    end
+
+    # hash hlist
+    def scan_all_hashs(limit: SCAN_LIMIT)
+      count = 0
+      name_start = ''
+      while true
+        names = @ssdb.hlist(name_start, '', limit: limit)
+        count += 1
+        puts "scan_all_hashs #{count} name_start: #{name_start}"
+        if names.size == 0
+          break
+        end
+
+        hash_len = []
+        @ssdb_redis.pipelined do |redis|
+          names.each do |name|
+            hash_len << redis.hlen(name)
+          end
+        end
+
+        short_names = []
+        long_names = []
+        names.each_with_index do |name, index|
+          if hash_len[index].value.to_i > 1000
+            long_names << name
+          else
+            short_names << name
+          end
+        end
+
+        scan_group_hashs(short_names)
+
+        long_names.each { |name| scan_one_hash(name) }
+
+        name_start = names.last
+      end
+    end
+
+    # hash hscan
+    def scan_group_hashs(names)
+      # 批量读取
+      result = []
+      @ssdb_redis.pipelined do |redis|
+        names.each do |name|
+          result << redis.hgetall(name)
+        end
+      end
+      # 批量写入
+      if @options[:write] == 'yes'
+        @pika.pipelined do |redis|
+          names.each_with_index do |name, index|
+            array = result[index].value.to_a
+            if array && array.size > 0
+              array.flatten!
+              redis.hmset name, *array
+            end
+          end
+        end
+      else
+        puts "result size #{result.size}"
+      end
+    end
+
+    # hash hscan
+    def scan_one_hash(name)
+      key_start = ''
+      while true
+        entries = @ssdb.hscan(name, key_start, '', limit: 1000)
+
+        if entries.size == 0
+          break
+        end
+
+        array = []
+        entries.each do |key, value|
+          array << key
+          array << value
+        end
+
+        if @options[:write] == 'yes'
+          @pika.hmset name, *array
+        else
+          puts "#{name} array size #{array.size}"
+        end
+
+        key_start = entries.last[0]
+      end
+    end
+
+    # zset zlist
+    def scan_all_zsets(limit: SCAN_LIMIT)
+      count = 0
+      name_start = ''
+      while true
+        names = @ssdb.zlist(name_start, '', limit: limit)
+
+        count += 1
+        puts "scan_all_zsets #{count} name_start: #{name_start}"
+
+        if names.size == 0
+          break
+        end
+
+        zset_len = []
+        @ssdb_redis.pipelined do |redis|
+          names.each do |name|
+            zset_len << redis.zcard(name)
+          end
+        end
+
+        short_names = []
+        long_names = []
+        names.each_with_index do |name, index|
+          if zset_len[index].value.to_i > 1000
+            long_names << name
+          else
+            short_names << name
+          end
+        end
+
+        scan_group_zsets(short_names)
+
+        long_names.each { |name| scan_one_zset(name) }
+
+        name_start = names.last
+      end
+    end
+
+    # zset zscan
+    def scan_group_zsets(names)
+      # 批量读取
+      result = []
+      @ssdb_redis.pipelined do |redis|
+        names.each do |name|
+          result << redis.zrange(name, 0, -1, with_scores: true)
+        end
+      end
+      # 批量写入
+      if @options[:write] == 'yes'
+        @pika.pipelined do |redis|
+          names.each_with_index do |name, index|
+            array = result[index].value.to_a
+            if array && array.size > 0
+              array.each { |pair| pair.reverse! }
+              redis.zadd name, array
+            end
+          end
+        end
+      else
+        puts "result size #{result.size}"
+      end
+    end
+
+    # zset zscan
+    def scan_one_zset(name)
+      score_start = ''
+      while true
+        entries = @ssdb.zscan(name, score_start, '', limit: SCAN_LIMIT)
+        if entries.size == 0
+          break
+        end
+
+        array = entries.map {|key, value| [value, key] }
+
+        if @options[:write] == 'yes'
+          @pika.zadd name, array
+        else
+          puts "#{name} array size #{array.size}"
+        end
+
+        score_start = entries.last[1].to_i + 1
+      end
+    end
+
+    # queue qlist
+    def scan_all_queues
+      count = 0
+      name_start = ''
+      while true
+        names = @ssdb.qlist(name_start, '', limit: 1000)
+        count += 1
+        puts "scan_all_queues #{count} name_start: #{name_start}"
+        if names.size == 0
+          break
+        end
+
+        names.each_with_index do |name, index|
+          scan_one_queue(name)
+          if index % 10 == 0
+            puts "process #{name} #{index}"
+          end
+        end
+        name_start = names.last
+      end
+    end
+
+    # queue qrange
+    def scan_one_queue(name)
+      array = @ssdb.qrange(name, 0, -1)
+      if array && array.size > 0
+        if @options[:write] == 'yes'
+          array.each do |item|
+            @pika.lpush name, item
+          end
+        else
+          puts "#{name} array size #{array.size}"
+        end
+      end
+    end
+  end
+end

data/lib/migrate_ssdb2pika/redis_ssdb_proxy.rb

@@ -0,0 +1,115 @@
+
+module MigrateSsdb2pika
+  class RedisSsdbProxy
+    attr_accessor :master, :slave
+
+    def initialize(master: nil, slave: nil, master_is_ssdb: true)
+      self.master = master
+      self.slave = slave
+
+      # 植入 ssdb? 方法
+      if master_is_ssdb
+        master.define_singleton_method(:ssdb?) { true }
+      else
+        master.define_singleton_method(:ssdb?) { false }
+      end
+    end
+
+    def clear_zset(key)
+      if master.ssdb?
+        master.zclear key
+        slave.del key
+      else
+        master.del key
+        slave.zclear key
+      end
+    end
+
+    def clear_hash(key)
+      if master.ssdb?
+        master.hclear key
+        slave.del key
+      else
+        master.del key
+        slave.hclear key
+      end
+    end
+
+    class << self
+      private
+
+      def send_to_slave(command)
+        class_eval <<-EOS
+          def #{command}(*args, &block)
+            slave.#{command}(*args, &block)
+          end
+        EOS
+      end
+
+      def send_to_master(command)
+        class_eval <<-EOS
+          def #{command}(*args, &block)
+            master.#{command}(*args, &block)
+          end
+        EOS
+      end
+
+      def send_to_both(command)
+        class_eval <<-EOS
+          def #{command}(*args, &block)
+            slave.#{command}(*args, &block)
+            master.#{command}(*args, &block)
+          end
+        EOS
+      end
+    end
+
+    send_to_master :dbsize
+    send_to_master :exists
+    send_to_master :get
+    send_to_master :getbit
+    send_to_master :getrange
+    send_to_master :hexists
+    send_to_master :hget
+    send_to_master :hgetall
+    send_to_master :hkeys
+    send_to_master :hlen
+    send_to_master :hmget
+    send_to_master :hvals
+    send_to_master :keys
+    send_to_master :lindex
+    send_to_master :llen
+    send_to_master :lrange
+    send_to_master :mget
+    send_to_master :randomkey
+    send_to_master :scard
+    send_to_master :sdiff
+    send_to_master :sinter
+    send_to_master :sismember
+    send_to_master :smembers
+    send_to_master :sort
+    send_to_master :srandmember
+    send_to_master :strlen
+    send_to_master :sunion
+    send_to_master :ttl
+    send_to_master :type
+    send_to_master :zcard
+    send_to_master :zcount
+    send_to_master :zrange
+    send_to_master :zrangebyscore
+    send_to_master :zrank
+    send_to_master :zrevrange
+    send_to_master :zscore
+
+    # all write opreate send to master slave both
+    def method_missing(name, *args, &block)
+      if master.respond_to?(name)
+        self.class.send(:send_to_both, name)
+        slave.send(name, *args, &block)
+        master.send(name, *args, &block)
+      else
+        super
+      end
+    end
+  end # RedisSsdbProxy
+end # MigrateSsdb2pika

data/lib/migrate_ssdb2pika/version.rb

@@ -0,0 +1,4 @@
+
+module MigrateSsdb2pika
+  VERSION = "0.1.0".freeze
+end

data/lib/migrate_ssdb2pika.rb

@@ -0,0 +1,2 @@
+
+require 'migrate_ssdb2pika'

data/migrate-ssdb2pika.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'migrate_ssdb2pika/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "migrate-ssdb2pika"
+  spec.version       = MigrateSsdb2pika::VERSION
+  spec.authors       = ["Vincent Xie"]
+  spec.email         = ["xiewenwei@gmail.com"]
+
+  spec.summary       = %q{迁移 SSDB 数据到 Redis 或 Pika 工具集}
+  spec.description   = %q{迁移 SSDB 数据到 Redis 或 Pika 工具集，提供迁移过程中的双写工具类和迁移历史数据工具}
+  spec.homepage      = "http://github.com/xiewenwei/migrate-ssdb2pika.git"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "bin"
+  spec.executables   = ["ssdb2pika"]
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "redis", ">= 3"
+
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification
+name: migrate-ssdb2pika
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Vincent Xie
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-03-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+description: 迁移 SSDB 数据到 Redis 或 Pika 工具集，提供迁移过程中的双写工具类和迁移历史数据工具
+email:
+- xiewenwei@gmail.com
+executables:
+- ssdb2pika
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/ssdb2pika
+- lib/migrate-ssdb2pika.rb
+- lib/migrate_ssdb2pika.rb
+- lib/migrate_ssdb2pika/migration.rb
+- lib/migrate_ssdb2pika/redis_ssdb_proxy.rb
+- lib/migrate_ssdb2pika/version.rb
+- migrate-ssdb2pika.gemspec
+homepage: http://github.com/xiewenwei/migrate-ssdb2pika.git
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: 迁移 SSDB 数据到 Redis 或 Pika 工具集
+test_files: []