embulk-plugin-redis-url 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bb328779e704f4e33b0411e31c82853c4638f707
+  data.tar.gz: 9348bb1c7a323d783679a2141a41ba8d393b239d
+SHA512:
+  metadata.gz: 65e36ae751469c2e37199c091f29c8e2d05d1fc5645d3d1c9f68c3b4fa86fd787fdee4b35800d2bad9bf9b3b6cca55cfe48a075e86e796368da1c61aca08a135
+  data.tar.gz: bbf9df820775e7ba898320f667a75060fec1ff5bc4af5400fe83e72b1a63135e24a2e6dbef3fb66bb1c6a50e5d94291a4e2e9dbc26c2bd3229d71cdd45f63d3b

data/Gemfile

@@ -0,0 +1,2 @@
+source "http://rubygems.org"
+gemspec

data/Gemfile.lock

@@ -0,0 +1,22 @@
+PATH
+  remote: .
+  specs:
+    embulk-plugin-redis-url (0.1.0)
+      redis (>= 3.0.5)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    rake (10.4.2)
+    redis (3.0.7)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0)
+  embulk-plugin-redis-url!
+  rake (>= 0.9.2)
+
+BUNDLED WITH
+   1.11.2

data/README.md

@@ -0,0 +1,32 @@
+# Embulk input/output plugin for Redis
+
+This [Embulk](https://github.com/embulk/embulk) output plugin writes records to a json column of a table.
+
+This plugin runs without transaction for now.
+
+## Configuration
+
+- **host** host name of the Redis server (string, default: "localhost")
+- **port** port of the Redis server (integer, default: 6379)
+- **db** destination database number (integer, default: 0)
+- **key_prefix** key prefix to search keys for input plugin (string)
+- **key** key name for output plugin (string, required)
+
+### Example
+
+```yaml
+out:
+  type: redis
+  host: localhost
+  port: 6379
+  db: 0
+  key: user_name
+
+in:
+  type: redis
+  host: localhost
+  port: 6379
+  db: 0
+  key_prefix: user_
+```
+

data/Rakefile

@@ -0,0 +1,4 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+task :default => [:build]

data/embulk-plugin-redis.gemspec

@@ -0,0 +1,21 @@
+
+Gem::Specification.new do |gem|
+  gem.name          = "embulk-plugin-redis-url"
+  gem.version       = "0.2.0"
+
+  gem.summary       = %q{Embulk plugin for Redis (with URL connection string support)}
+  gem.description   = gem.summary
+  gem.authors       = ["Mitsunori Komatsu", "Tom Maiaroto"]
+  gem.email         = ["komamitsu@gmail.com", "tom@funnelenvy.com"]
+  gem.license       = "Apache 2.0"
+  gem.homepage      = "https://github.com/FunnelEnvy/embulk-plugin-redis"
+
+  gem.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.has_rdoc      = false
+
+  gem.add_dependency 'redis', ['>= 3.0.5']
+  gem.add_development_dependency 'bundler', ['~> 1.0']
+  gem.add_development_dependency 'rake', ['>= 0.9.2']
+end

data/lib/embulk/input_redis.rb

@@ -0,0 +1,50 @@
+module Embulk
+
+  class InputRedis < InputPlugin
+    require 'redis'
+
+    Plugin.register_input('redis', self)
+
+    def self.transaction(config, &control)
+      task = {
+        'host' => config.param('host', :string, :default => 'localhost'),
+        'port' => config.param('port', :int, :default => 6379),
+        'db' => config.param('db', :int, :default => 0),
+        'key_prefix' => config.param('key_prefix', :string, :default => ''),
+        'url' => config.param('url', :string),
+      }
+      threads = config.param('threads', :int, default: 1)
+
+      columns = [
+        Column.new(0, 'key', :string),
+        Column.new(1, 'value', :string),
+      ]
+
+      puts "Redis input started."
+      commit_reports = yield(task, columns, threads)
+      puts "Redis input finished. Commit reports = #{commit_reports.to_json}"
+
+      return {}
+    end
+
+    def run
+      puts "Redis input thread #{@index}..."
+
+      if @task['url'].nil? || @task['url'].empty?
+        r = ::Redis.new(:host => @task['host'], :port => @task['port'], :db => @task['db'])
+      else
+        r = ::Redis.new(:url => @task['url'])
+      end
+      
+      r.keys("#{@task['key_prefix']}*").each do |k|
+        @page_builder.add([k, r.get(k)])
+      end
+      @page_builder.finish  # don't forget to call finish :-)
+
+      commit_report = {
+      }
+      return commit_report
+    end
+  end
+
+end

data/lib/embulk/output_redis.rb

@@ -0,0 +1,61 @@
+module Embulk
+
+  class OutputRedis < OutputPlugin
+    require 'redis'
+
+    Plugin.register_output('redis', self)
+
+    def self.transaction(config, schema, processor_count, &control)
+      task = {
+        'host' => config.param('host', :string, :default => 'localhost'),
+        'port' => config.param('port', :int, :default => 6379),
+        'db' => config.param('db', :int, :default => 0),
+        'key' => config.param('key', :string),
+        'url' => config.param('url', :string),
+      }
+
+      puts "Redis output started."
+      commit_reports = yield(task)
+      puts "Redis output finished. Commit reports = #{commit_reports.to_json}"
+
+      return {}
+    end
+
+    def initialize(task, schema, index)
+      puts "Example output thread #{index}..."
+      super
+      @records = 0
+      if task['url'].nil? || task['url'].empty?
+        @redis = ::Redis.new(:host => task['host'], :port => task['port'], :db => task['db'])
+      else
+        @redis = ::Redis.new(:url => task['url'])
+      end
+    end
+
+    def close
+    end
+
+    def add(page)
+      page.each do |record|
+        hash = Hash[schema.names.zip(record)]
+        puts "#{@message}: #{hash.to_json}"
+        @redis.set(hash[task['key']], hash)
+        @records += 1 
+      end
+    end
+
+    def finish
+    end
+
+    def abort
+    end
+
+    def commit
+      commit_report = {
+        "records" => @records
+      }
+      return commit_report
+    end
+  end
+
+end

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification
+name: embulk-plugin-redis-url
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Mitsunori Komatsu
+- Tom Maiaroto
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-04-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.5
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: Embulk plugin for Redis (with URL connection string support)
+email:
+- komamitsu@gmail.com
+- tom@funnelenvy.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- embulk-plugin-redis.gemspec
+- lib/embulk/input_redis.rb
+- lib/embulk/output_redis.rb
+homepage: https://github.com/FunnelEnvy/embulk-plugin-redis
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Embulk plugin for Redis (with URL connection string support)
+test_files: []