bricolage 5.14.0 → 5.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5bb93d5d2f2b8100f394fefef4df4a0312e612a4
-  data.tar.gz: a961533ace5a56216757ab3c99666780922511ac
+  metadata.gz: f55cd961f8a67911b9ceb8383202dbf5a4bc5eac
+  data.tar.gz: 5ea072313fd1a0a7455a57e02ae8a38579c1d8ef
 SHA512:
-  metadata.gz: 7d5e13b4683b83f850ebca170c0ba4f00917874f386d95d16b5764d63625be388de3940a3f8be6b07a19bb000b9da3c024aa05c4795d48067efc81b672ada610
-  data.tar.gz: 68d542616e8a6826c0e9bddc33afd9744b449c7ee218610d65750186915ec51913f8144c8155bb083feb80b17ee086eafdc5c7dec07f16db012912d8546c5fba
+  metadata.gz: 43b49955f3739925b6c92e84f72fde9457d2f72a10034316f2fec0a44cbde62cdfd9072ae29048023092a3662bcae16733339a72429061159898740db2f3e15f
+  data.tar.gz: 397a51bc3f114ca552ee3441992ef7311e38c5cb081cde01acece6d2e0daa2db7cb8c35f7a3d7ab374c40dd6ab9123f05ffe389efc0b5d7a6cec281dc4188642

data/jobclass/redis-export.rb

@@ -0,0 +1,40 @@
+require 'bricolage/psqldatasource'
+require 'bricolage/redisdatasource'
+require 'redis'
+
+JobClass.define('redis-export') {
+  parameters {|params|
+    # Export
+    params.add DataSourceParam.new('psql', 'src-ds', 'Source data source.')
+    params.add SrcTableParam.new(optional: false)
+    params.add SQLFileParam.new(optional: true)
+
+    # Redis import
+    params.add DataSourceParam.new('redis', 'dest-ds', 'Redis cluster')
+    params.add StringParam.new('key-column', 'REDIS_KEY', 'Redis object key. default: id', optional: true)
+    params.add StringParam.new('prefix', 'REDIS_PREFIX', 'Redis object key prefix', optional: true)
+    params.add StringParam.new('encode', 'REDIS_ENCODE', 'Redis object encoding. default: hash', optional: true)
+  }
+
+  script {|params, script|
+    # Export
+    script.task(params['dest-ds']) {|task|
+      task.import params['src-ds'],
+        params['src-tables'].first,
+        sql_statement(params),
+        params['key-column'] || "id",
+        params['prefix'],
+        params['encode'] || "hash"
+    }
+  }
+
+  def sql_statement(params)
+    return params['sql-file'] if params['sql-file']
+    srcs = params['src-tables']
+    raise ParameterError, "src-tables must be singleton when no sql-file is given" unless srcs.size == 1
+    src_table_var = srcs.keys.first
+    stmt = SQLStatement.for_string("select * from $#{src_table_var};")
+    stmt.declarations = Declarations.new({src_table_var => src_table_var})
+    stmt
+  end
+}

data/jobclass/streaming_load.rb

@@ -134,10 +134,12 @@ class StreamingLoadJobClass < RubyJobClass
       end
       create_load_log_file(objects) {|log_url|
         @ds.open {|conn|
-          execute_update conn, copy_load_log_stmt(log_url, @src.credential_string)
-          foreach_loaded_object(objects) do |obj|
-            obj.dequeue(@noop)
-          end
+          create_tmp_log_table(conn, log_url) {|tmp_log_table|
+            loaded, not_loaded = partition_loaded_objects(conn, objects, tmp_log_table)
+            loaded.each do |obj|
+              obj.dequeue(force: true, noop: @noop)
+            end
+          }
         }
       }
     end

data/lib/bricolage/postgresconnection.rb

@@ -28,6 +28,11 @@ module Bricolage
       raise PostgreSQLException.wrap(ex)
     end
 
+    def execute_query(query, &block)
+      @logger.info "[#{@ds.name}] #{query}"
+      exec(query, &block)
+    end
+
     alias update execute
 
     def drop_table(name)
@@ -43,12 +48,7 @@ module Bricolage
     def select(table, &block)
       query = "select * from #{table}"
       @logger.info "[#{@ds.name}] #{query}"
-      rs = @connection.exec(query)
-      begin
-        yield rs
-      ensure
-        rs.clear
-      end
+      exec(query, &block)
     end
 
     def vacuum(table)
@@ -73,6 +73,20 @@ module Bricolage
       t = e - b
       @logger.info "#{'%.1f' % t} secs"
     end
+    
+    def exec(query, &block)
+      @connection.send_query(query)
+      @connection.set_single_row_mode
+      loop do
+        rs = @connection.get_result or break
+        begin
+          rs.check
+          yield rs
+        ensure
+          rs.clear
+        end
+      end
+    end
   end
 
 end

data/lib/bricolage/psqldatasource.rb

@@ -103,7 +103,11 @@ module Bricolage
     end
 
     def query(query)
-      open {|conn| conn.query(query) }
+      open {|conn| conn.execute(query) }
+    end
+    
+    def execute_query(query, &block)
+      open {|conn| conn.execute_query(query, &block) }
     end
 
     def drop_table(name)

data/lib/bricolage/redisdatasource.rb

@@ -0,0 +1,112 @@
+require 'bricolage/datasource'
+require 'bricolage/commandutils'
+require 'redis'
+require 'json'
+
+module Bricolage
+
+  class RedisDataSource < DataSource
+    declare_type 'redis'
+
+    def initialize(host: 'localhost', port: 6380, **opts)
+      @host = host
+      @port = port
+      @options = opts
+    end
+
+    attr_reader :host
+    attr_reader :port
+    attr_reader :opts
+
+    def new_task
+      RedisTask.new(self)
+    end
+
+    def client
+      @client = @client || Redis.new(:host => @host, :port => @port, **@options)
+    end
+  end
+
+  class RedisTask < DataSourceTask
+    def import(src, table, query, key_column, prefix, encode)
+      add Import.new(src, table, query, key_column, prefix, encode)
+    end
+
+    class Import < Action
+      def initialize(src, table, query, key_column, prefix, encode)
+        @src = src
+        @table = table
+        @query = query
+        @key_column = key_column
+        puts key_column
+        @prefix = prefix
+        @encode = encode
+        @read_count = 0
+        @write_count = 0
+      end
+
+      def bind(*args)
+        @query.bind(*args)
+      end
+
+      def source
+        @query.stripped_source
+      end
+
+      def prefix
+        @prefix = @prefix || "#{@table.last.schema}_#{@table.last.name}_"
+      end
+
+      def import
+        read_row do |row|
+          write_row(row)
+        end
+      end
+
+      def read_row
+        @src.execute_query(source) do |rs|
+          rs.each do |row|
+            yield row
+            @read_count += 1
+            ds.logger.info "Rows read: #{@read_count}" if @read_count % 100000 == 0
+          end
+        end
+      end
+
+      def write_row(row, &block)
+        key = key(row)
+        case @encode
+        when 'hash'
+          # set a value for each key:field pair
+          r = []
+          row.each do |field,value|
+            r.push ds.client.hset(key, field, value)
+          end
+        when 'json'
+          r = ds.client.set(key, JSON.generate(row))
+        else
+          raise %Q("encode: #{type}" is not supported)
+        end
+        yield r if block
+        ds.logger.info "Key sample: #{key}" if @write_count == 0
+        @write_count += 1
+      end
+
+      def key(row)
+        key_columns = @key_column.split(',').map(&:strip).map {|k| row[k]}
+        prefix + key_columns.join('_')
+      end
+
+      def run
+        begin
+          import
+        rescue => ex
+          ds.logger.error ex.backtrace.join("\n")
+          raise JobFailure, ex.message
+        end
+        ds.logger.info "Rows written: #{@write_count}"
+        JobResult.success
+      end
+    end
+  end
+end

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.14.0'
+  VERSION = '5.15.0'
 end

data/test/home/Gemfile.lock

@@ -1,54 +1,56 @@
 PATH
   remote: ../..
   specs:
-    bricolage (5.14.0)
+    bricolage (5.15.0)
       aws-sdk (~> 2)
       mysql2
       pg
+      redis (>= 3.0.0)
       td
 
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-sdk (2.2.13)
-      aws-sdk-resources (= 2.2.13)
-    aws-sdk-core (2.2.13)
+    aws-sdk (2.2.26)
+      aws-sdk-resources (= 2.2.26)
+    aws-sdk-core (2.2.26)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.2.13)
-      aws-sdk-core (= 2.2.13)
+    aws-sdk-resources (2.2.26)
+      aws-sdk-core (= 2.2.26)
     coderay (1.1.0)
     fluent-logger (0.5.1)
       msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
     hirb (0.7.3)
-    httpclient (2.5.3.3)
+    httpclient (2.7.1)
     jmespath (1.1.3)
     json (1.8.3)
     method_source (0.8.2)
-    msgpack (0.5.11)
-    mysql2 (0.4.2)
+    msgpack (0.5.12)
+    mysql2 (0.4.3)
     parallel (0.6.5)
     pg (0.18.4)
     pry (0.10.3)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
+    redis (3.2.2)
     ruby-progressbar (1.7.5)
     rubyzip (1.1.7)
     slop (3.6.0)
-    td (0.13.1)
+    td (0.13.2)
       hirb (>= 0.4.5)
-      msgpack (>= 0.4.4, < 0.5.12, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
+      msgpack (>= 0.4.4, < 0.8.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
       parallel (~> 0.6.1)
       ruby-progressbar (~> 1.7.5)
       rubyzip (~> 1.1.7)
-      td-client (~> 0.8.77)
+      td-client (~> 0.8.78)
       td-logger (~> 0.3.21)
       yajl-ruby (~> 1.1)
       zip-zip (~> 0.3)
-    td-client (0.8.77)
-      httpclient (>= 2.5.2, < 2.6.0)
+    td-client (0.8.78)
+      httpclient (~> 2.7)
       json (>= 1.7.6)
-      msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
+      msgpack (>= 0.4.4, < 0.8.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
     td-logger (0.3.25)
       fluent-logger (~> 0.5.0)
       msgpack (>= 0.4.4, < 0.8.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)

data/test/home/subsys/redis_export.job

@@ -0,0 +1,6 @@
+class: redis-export
+src-ds: sql
+src-tables:
+    users: $test_schema.users
+dest-ds: redis
+key-column: id, user_name

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.14.0
+  version: 5.15.0
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-25 00:00:00.000000000 Z
+date: 2016-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -116,6 +130,7 @@ files:
 - jobclass/noop.rb
 - jobclass/rebuild-drop.rb
 - jobclass/rebuild-rename.rb
+- jobclass/redis-export.rb
 - jobclass/s3-put.rb
 - jobclass/sql.rb
 - jobclass/streaming_load.rb
@@ -146,6 +161,7 @@ files:
 - lib/bricolage/parameters.rb
 - lib/bricolage/postgresconnection.rb
 - lib/bricolage/psqldatasource.rb
+- lib/bricolage/redisdatasource.rb
 - lib/bricolage/resource.rb
 - lib/bricolage/rubyjobclass.rb
 - lib/bricolage/s3datasource.rb
@@ -192,6 +208,7 @@ files:
 - test/home/subsys/raw-vacuum.jobnet
 - test/home/subsys/raw-vacuum.sql.job
 - test/home/subsys/rebuild.sql.job
+- test/home/subsys/redis_export.job
 - test/home/subsys/search_backends.ct
 - test/home/subsys/separated.job
 - test/home/subsys/separated.sql
@@ -224,7 +241,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: SQL batch framework