bricolage 5.15.0 → 5.15.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f55cd961f8a67911b9ceb8383202dbf5a4bc5eac
-  data.tar.gz: 5ea072313fd1a0a7455a57e02ae8a38579c1d8ef
+  metadata.gz: 26898f98eb266df22a4940e128b1ae4cf44afd83
+  data.tar.gz: fd57e7f63e899b6f9f560a2694a4e1466efcb655
 SHA512:
-  metadata.gz: 43b49955f3739925b6c92e84f72fde9457d2f72a10034316f2fec0a44cbde62cdfd9072ae29048023092a3662bcae16733339a72429061159898740db2f3e15f
-  data.tar.gz: 397a51bc3f114ca552ee3441992ef7311e38c5cb081cde01acece6d2e0daa2db7cb8c35f7a3d7ab374c40dd6ab9123f05ffe389efc0b5d7a6cec281dc4188642
+  metadata.gz: 46996f94e09c5e9c2efd6814492ebaff39fbdfc1b765a906dbd71dc387b6fecc6bd446a6cd9d916be2479bc0f83588380c3f91afb985ee9ad29f0275c63ed779
+  data.tar.gz: fad034727fa9b0781eeb642bda3da779ff8183ea94e7f7eef9d5d5e049a41075c309c8585fcdc37784730fed6713eac26e7979b74f0a2532ce01da83fc96b989

data/jobclass/redis-export.rb

@@ -14,6 +14,7 @@ JobClass.define('redis-export') {
     params.add StringParam.new('key-column', 'REDIS_KEY', 'Redis object key. default: id', optional: true)
     params.add StringParam.new('prefix', 'REDIS_PREFIX', 'Redis object key prefix', optional: true)
     params.add StringParam.new('encode', 'REDIS_ENCODE', 'Redis object encoding. default: hash', optional: true)
+    params.add StringParam.new('expire', 'REDIS_TTL', 'Redis object TTL. default: none', optional: true)
   }
 
   script {|params, script|
@@ -24,7 +25,8 @@ JobClass.define('redis-export') {
         sql_statement(params),
         params['key-column'] || "id",
         params['prefix'],
-        params['encode'] || "hash"
+        params['encode'] || "hash",
+        expire: params['expire'].to_i
     }
   }
 

data/lib/bricolage/postgresconnection.rb

@@ -1,4 +1,5 @@
 require 'bricolage/exception'
+require 'securerandom'
 require 'pg'
 
 module Bricolage
@@ -10,6 +11,7 @@ module Bricolage
       @connection = connection
       @ds = ds
       @logger = logger
+      @cursor = nil
     end
 
     def source
@@ -33,6 +35,36 @@ module Bricolage
       exec(query, &block)
     end
 
+    def execute_query_with_cursor(query, fetch_size, cursor, &block)
+      raise "Begin transaction before invoking this method" unless in_transaction?
+      if @cursor.nil?
+        @cursor = cursor || (0...32).map { alphabets[rand(alphabets.length)] }.join
+        declare_cursor = "declare #{@cursor} cursor for #{query}"
+        @logger.info "[#{@ds.name}] #{declare_cursor}"
+        @connection.exec(declare_cursor)
+      elsif !@cursor.nil? && cursor.nil?
+        raise "Cursor in use"
+      elsif @cursor != cursor
+        raise "Invalid cursor"
+      end
+      fetch = "fetch #{fetch_size} in #{@cursor}"
+      @logger.info "[#{@ds.name}] #{fetch}" if cursor.nil?
+      yield @connection.exec(fetch)
+      return @cursor
+    end
+
+    def clear_cursor
+      @cursor = nil
+    end
+
+    def alphabets
+      @alphabets = @alphabets || [('a'...'z'), ('A'...'Z')].map { |i| i.to_a }.flatten
+    end
+
+    def in_transaction?
+      @connection.transaction_status == PG::Constants::PQTRANS_INTRANS
+    end
+
     alias update execute
 
     def drop_table(name)
@@ -88,5 +120,4 @@ module Bricolage
       end
     end
   end
-
 end

data/lib/bricolage/psqldatasource.rb

@@ -110,6 +110,28 @@ module Bricolage
       open {|conn| conn.execute_query(query, &block) }
     end
 
+    def cursor_transaction(&block)
+      raise "Cursor in use" if cursor_in_transaction?
+      conn = PG::Connection.open(host: @host, port: @port, dbname: @database, user: @user, password: password)
+      @cur_conn = PostgresConnection.new(conn, self, logger)
+      @cur_conn.execute("begin transaction")
+      yield
+    ensure
+      @cur_conn.execute("commit") if cursor_in_transaction?
+      @cur_conn.clear_cursor if @cur_conn
+      @cur_conn = nil
+      conn.close if conn
+    end
+
+    def cursor_execute_query(query, fetch_size: 10000, cursor: nil, &block)
+      raise "Begin transaction before invoking this method" unless cursor_in_transaction?
+      @cur_conn.execute_query_with_cursor(query, fetch_size, cursor, &block)
+    end
+
+    def cursor_in_transaction?
+      @cur_conn && @cur_conn.in_transaction?
+    end
+
     def drop_table(name)
       open {|conn| conn.drop_table(name) }
     end

data/lib/bricolage/redisdatasource.rb

@@ -28,19 +28,19 @@ module Bricolage
   end
 
   class RedisTask < DataSourceTask
-    def import(src, table, query, key_column, prefix, encode)
-      add Import.new(src, table, query, key_column, prefix, encode)
+    def import(src, table, query, key_column, prefix, encode, expire: nil)
+      add Import.new(src, table, query, key_column, prefix, encode, expire)
     end
 
     class Import < Action
-      def initialize(src, table, query, key_column, prefix, encode)
+      def initialize(src, table, query, key_column, prefix, encode, expire)
         @src = src
         @table = table
         @query = query
-        @key_column = key_column
-        puts key_column
+        @key_columns = key_column.split(',').map(&:strip)
         @prefix = prefix
         @encode = encode
+        @expire = expire
         @read_count = 0
         @write_count = 0
       end
@@ -58,48 +58,76 @@ module Bricolage
       end
 
       def import
-        read_row do |row|
-          write_row(row)
-        end
+        results = []
+        @src.cursor_transaction {
+          read_count = 0
+          loop do
+            futures = []
+            ds.client.pipelined do 
+              read_count = read_row do |row|
+                futures.push write_row row
+              end
+            end
+            results.push futures.flatten.map {|f| f.value}.all?
+            break if read_count == 0
+          end
+        }
+        @cursor = nil
+        results.all?
       end
 
       def read_row
-        @src.execute_query(source) do |rs|
+        rs_size = 0
+        @cursor = @src.cursor_execute_query(source, cursor: @cursor) do |rs|
+          rs_size = rs.values.size
+          break if rs_size == 0
           rs.each do |row|
             yield row
             @read_count += 1
             ds.logger.info "Rows read: #{@read_count}" if @read_count % 100000 == 0
           end
         end
+        rs_size
       end
 
-      def write_row(row, &block)
+      def write_row(row)
         key = key(row)
+        data = delete_key_columns(row)
+        f = []
         case @encode
         when 'hash'
           # set a value for each key:field pair
-          r = []
-          row.each do |field,value|
-            r.push ds.client.hset(key, field, value)
+          data.each do |field,value|
+            f.push ds.client.hset(key, field, value)
           end
         when 'json'
-          r = ds.client.set(key, JSON.generate(row))
+          f.push ds.client.set(key, JSON.generate(data))
         else
           raise %Q("encode: #{type}" is not supported)
         end
-        yield r if block
-        ds.logger.info "Key sample: #{key}" if @write_count == 0
+        f.push ds.client.expire(key, @expire) if @expire
         @write_count += 1
+        return f
+      end
+
+      def delete_key_columns(row)
+        r = row.dup
+        @key_columns.each do |key|
+          r.delete(key)
+        end
+        r.empty? ? {1 => 1} : r
       end
 
       def key(row)
-        key_columns = @key_column.split(',').map(&:strip).map {|k| row[k]}
-        prefix + key_columns.join('_')
+        prefix + @key_columns.map {|k| row[k]}.join('_')
       end
 
       def run
         begin
-          import
+          ds.logger.info "Key Pattern: #{prefix}<#{@key_columns.join('_')}>"
+          ds.logger.info "Encode: #{@encode}"
+          ds.logger.info "Expire: #{@expire}"
+          raise "Unexpected error. Please check data." unless import
         rescue => ex
           ds.logger.error ex.backtrace.join("\n")
           raise JobFailure, ex.message

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.15.0'
+  VERSION = '5.15.1'
 end

data/test/home/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ../..
   specs:
-    bricolage (5.15.0)
+    bricolage (5.15.1)
       aws-sdk (~> 2)
       mysql2
       pg

data/test/home/subsys/redis_export.job

@@ -1,6 +1,9 @@
 class: redis-export
 src-ds: sql
 src-tables:
-    users: $test_schema.users
+    user_cook_recipes: $test_schema.user_cook_recipes
 dest-ds: redis
-key-column: id, user_name
+prefix: ha_ucp_
+key-column: user_id, recipe_id
+encode: hash
+expire: 604800

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.15.0
+  version: 5.15.1
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-15 00:00:00.000000000 Z
+date: 2016-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg