bricolage-redis 5.26.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c8b0dd1f32e3cda4a5638cfcd08db7061a121223
+  data.tar.gz: 0c4bb9f8f7d1ebe9d44797362680ed064afa3646
+SHA512:
+  metadata.gz: 87f635b937a9d1e00f4d9a187f3eb49b2e7938dc1f15f6bfba3585bd81891079ff55e986739bf2bf29abc564b4b2dfd3c084498d1abba5487b0184e41bb27fc9
+  data.tar.gz: 92f2ac32c7a911312356e7fb7fbaa6504cf4289250676ab889fe4e92ea8c23e6c104884c6b6f182c91da84cd4fdd49d8d2102d7de9649dcf3ff0edae0b337351

data/README.md

@@ -0,0 +1,25 @@
+# bricolage-redis
+
+Redis-related job classes for Bricolage batch framework.
+
+This software is written in working time in Cookpad, Inc.
+
+## Home Page
+
+https://github.com/bricolages/bricolage-redis
+
+## Usage
+
+Add following line to your Gemfile:
+```
+gem 'bricolage-redis', require: 'bricolage-redis'
+```
+
+## License
+
+MIT license.
+SEe LICENSES file for details.
+
+## Author
+
+Minero Aoki

data/RELEASE.md

@@ -0,0 +1,5 @@
+# bricolage-redis release note
+
+## version 5.26.0
+
+- first release.

data/jobclass/redis-export.rb

@@ -0,0 +1,42 @@
+require 'bricolage/psqldatasource'
+require 'bricolage/redisdatasource'
+require 'redis'
+
+JobClass.define('redis-export') {
+  parameters {|params|
+    # Export
+    params.add DataSourceParam.new('psql', 'src-ds', 'Source data source.')
+    params.add SrcTableParam.new(optional: false)
+    params.add SQLFileParam.new(optional: true)
+
+    # Redis import
+    params.add DataSourceParam.new('redis', 'dest-ds', 'Redis cluster')
+    params.add StringParam.new('key-column', 'REDIS_KEY', 'Redis object key. default: id', optional: true)
+    params.add StringParam.new('prefix', 'REDIS_PREFIX', 'Redis object key prefix', optional: true)
+    params.add StringParam.new('encode', 'REDIS_ENCODE', 'Redis object encoding. default: hash', optional: true)
+    params.add StringParam.new('expire', 'REDIS_TTL', 'Redis object TTL. default: none', optional: true)
+  }
+
+  script {|params, script|
+    # Export
+    script.task(params['dest-ds']) {|task|
+      task.import params['src-ds'],
+        params['src-tables'].first,
+        sql_statement(params),
+        params['key-column'] || "id",
+        params['prefix'],
+        params['encode'] || "hash",
+        expire: params['expire'].to_i
+    }
+  }
+
+  def sql_statement(params)
+    return params['sql-file'] if params['sql-file']
+    srcs = params['src-tables']
+    raise ParameterError, "src-tables must be singleton when no sql-file is given" unless srcs.size == 1
+    src_table_var = srcs.keys.first
+    stmt = SQLStatement.for_string("select * from $#{src_table_var};")
+    stmt.declarations = Declarations.new({src_table_var => src_table_var})
+    stmt
+  end
+}

data/lib/bricolage-redis.rb

@@ -0,0 +1,5 @@
+require 'bricolage/jobclass'
+require 'pathname'
+
+jobclass_path = Pathname(__dir__).realpath.parent.cleanpath + 'jobclass'
+Bricolage::JobClass.add_load_path jobclass_path

data/lib/bricolage/redisdatasource.rb

@@ -0,0 +1,153 @@
+require 'bricolage/datasource'
+require 'bricolage/commandutils'
+require 'redis'
+require 'json'
+
+module Bricolage
+
+  class RedisDataSource < DataSource
+    declare_type 'redis'
+
+    def initialize(host: 'localhost', port: 6380, **options)
+      @host = host
+      @port = port
+      @options = options
+    end
+
+    attr_reader :host
+    attr_reader :port
+    attr_reader :options
+
+    def new_task
+      RedisTask.new(self)
+    end
+
+    def open
+      client = Redis.new(host: @host, port: @port, **@options)
+      yield client
+    end
+  end
+
+  class RedisTask < DataSourceTask
+    def import(src, table, query, key_column, prefix, encode, expire: nil)
+      add Import.new(src, table, query, key_column, prefix, encode, expire)
+    end
+
+    class Import < Action
+      def initialize(src, table, query, key_column, prefix, encode, expire)
+        @src = src
+        @table = table
+        @query = query
+        @key_columns = key_column.split(',').map(&:strip)
+        @prefix = prefix || "#{@table.last.schema}_#{@table.last.name}_"
+        @encode = encode
+        @expire = expire
+      end
+
+      def bind(*args)
+        @query.bind(*args)
+      end
+
+      def source
+        @query.stripped_source
+      end
+
+      def run
+        logger = ds.logger
+        begin
+          logger.info "Key Pattern: #{@prefix}<#{@key_columns.join('_')}>"
+          logger.info "Encode: #{@encode}"
+          logger.info "Expire: #{@expire}"
+          ds.open {|client|
+            writer = RedisRowWriter.for_encode(@encode).new(client, @prefix, @key_columns)
+            import writer
+          }
+        rescue => ex
+          logger.exception ex
+          raise JobFailure, ex.message
+        end
+        JobResult.success
+      end
+
+      BATCH_SIZE = 5000
+
+      def import(writer)
+        count = 0
+        @src.query_batch(source, BATCH_SIZE) do |rs|
+          writer.pipelined {
+            rs.each do |row|
+              writer.write(row)
+              count += 1
+              ds.logger.info "transfered: #{count} rows" if count % 100_0000 == 0
+            end
+          }
+        end
+        ds.logger.info "all rows written: #{count} rows"
+      end
+    end
+  end
+
+  class RedisRowWriter
+    def RedisRowWriter.for_encode(encode)
+      case encode
+      when 'hash' then RedisHashRowWriter
+      when 'json' then RedisJSONRowWriter
+      else
+        raise ParameterError, "unsupported Redis encode: #{encode.inspect}"
+      end
+    end
+
+    def initialize(client, prefix, key_columns)
+      @client = client
+      @prefix = prefix
+      @key_columns = key_columns
+    end
+
+    attr_reader :prefix
+    attr_reader :write_count
+
+    def key(row)
+      @prefix + @key_columns.map {|k| row[k] }.join('_')
+    end
+
+    def value_columns(row)
+      r = row.dup
+      @key_columns.each do |key|
+        r.delete(key)
+      end
+      r.empty? ? {1 => 1} : r
+    end
+
+    def pipelined(&block)
+      @client.pipelined(&block)
+    end
+
+    def write(row)
+      key = key(row)
+      futures = do_write(key, value_columns(row))
+      futures.push @client.expire(key, @expire) if @expire
+      futures
+    end
+
+    def expire
+      @client.expire(key, @expire)
+    end
+  end
+
+  class RedisHashRowWriter < RedisRowWriter
+    def do_write(key, values)
+      # set a value for each key:field pair
+      values.map {|field, value|
+        @client.hset(key, field, value)
+      }
+    end
+  end
+
+  class RedisJSONRowWriter < RedisRowWriter
+    def do_write(key, values)
+      future = @client.set(key, JSON.generate(values))
+      [future]
+    end
+  end
+
+end

metadata

@@ -0,0 +1,76 @@
+--- !ruby/object:Gem::Specification
+name: bricolage-redis
+version: !ruby/object:Gem::Version
+  version: 5.26.0
+platform: ruby
+authors:
+- Minero Aoki
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-01-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bricolage
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.26.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.26.0
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+description: 
+email: aamine@loveruby.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- RELEASE.md
+- jobclass/redis-export.rb
+- lib/bricolage-redis.rb
+- lib/bricolage/redisdatasource.rb
+homepage: https://github.com/bricolages/bricolage-redis
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: Redis-related job classes for Bricolage batch framework
+test_files: []