bricolage 5.20.4 → 5.20.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 26539773f2494fbfad668fc6865ea15d8e6e42be
-  data.tar.gz: bad48dfdce6017bde0e1a493b9075812e3155d66
+  metadata.gz: 54eb8ff011e173d9091a2945ef189097c5fc055e
+  data.tar.gz: c68f93bf30468fdcb221c09d0c81e43d8f9a58be
 SHA512:
-  metadata.gz: 839058660738526f1dc04b9da384ab84207885da21da8c388fb9397e36122fac517b07da8aad3b183525aaa353fb787c00c505f68e312cef6a13de02473949c0
-  data.tar.gz: 0b81d23d4dad5bb8b51e0f7200bb980e8b6de2647c81bed70886bb87d77351a885c241819a5cf81ed4da6bf084794a92163feadb7454a0dbc5041e1695e81cb4
+  metadata.gz: aa8293db93a8df5d17dc69827d6ad4f18638e6dfb0e79af33a40f26cebced6d4b682f83078df8105f512909d58f10513f292f2e1197940b25244dcbee975691a
+  data.tar.gz: 3f2ddade13d2cb6662624c91fde4fb1223eeeeba084256c5008084867577e4ef7c04f5e628c66a98200c92c907ddb0565207a9b0f3224bf55338a55012d00b54

data/lib/bricolage/mysqldatasource.rb

@@ -100,7 +100,7 @@ module Bricolage
       end
 
       def export_by_sqldump
-        cmds = [[sqldump_path.to_s, "--#{@format}", ds.host, ds.port.to_s, ds.username, ds.password, ds.database, @statement.stripped_source]]
+        cmds = [[{"SQLDUMP_PASSWORD" => ds.password}, sqldump_path.to_s, "--#{@format}", ds.host, ds.port.to_s, ds.username, ds.database, @statement.stripped_source]]
         cmds.push [GZIP_COMMAND] if @gzip
         cmds.last.push({out: @path.to_s})
         ds.logger.info '[CMD] ' + format_pipeline(cmds)
@@ -108,14 +108,13 @@ module Bricolage
         statuses.each_with_index do |st, idx|
           unless st.success?
             cmd = cmds[idx].first
-            raise JobFailure, "#{cmd} failed (status #{st.to_i})"
+            raise JobFailure, "sqldump failed (status #{st.to_i})"
           end
         end
       end
 
       def format_pipeline(cmds)
-        cmds = cmds.map {|args| args.dup }
-        cmds.first[5] = '****'
+        cmds = cmds.map {|args| args[0].kind_of?(Hash) ? args[1..-1] : args.dup }   # do not show env
         cmds.map {|args| %Q("#{args.join('" "')}") }.join(' | ')
       end
 
@@ -231,22 +230,44 @@ module Bricolage
       end
 
       def s3export
-        cmd = build_cmd(command_parameters)
-        ds.logger.info '[CMD] ' + cmd.join(' ')
+        cmd = build_cmd(environment_variables, command_parameters)
+        ds.logger.info "[CMD] #{format_cmd(cmd)}"
         Open3.popen2e(*cmd) do |input, output, thread|
           input.close
           output.each do |line|
             puts line
           end
           unless thread.value.success?
-            raise JobFailure, "#{cmd.join(' ')} failed (status #{thread.value.to_i})"
+            raise JobFailure, "mys3dump failed (status #{thread.value.to_i})"
           end
         end
       end
 
+      def format_cmd(cmd)
+        cmd[1..-1].join(' ')   # do not show env
+      end
+
+      def environment_variables
+        {
+          'AWS_ACCESS_KEY_ID' => @s3ds.access_key,
+          'AWS_SECRET_ACCESS_KEY' => @s3ds.secret_key,
+          'MYS3DUMP_PASSWORD' => ds.password
+        }
+      end
+
       def command_parameters
-        params = {jar: mys3dump_path.to_s, h: ds.host, P: ds.port.to_s, D: ds.database, u: ds.username, p: ds.password, o: connection_property, t: @table,
-                   'Daws.accessKeyId' => @s3ds.access_key, 'Daws.secretKey' => @s3ds.secret_key, b: @s3ds.bucket.name, x: @prefix}
+        params = {
+          jar: mys3dump_path.to_s,
+          h: ds.host,
+          P: ds.port.to_s,
+          D: ds.database,
+          u: ds.username,
+          #p: ds.password,
+          o: connection_property,
+          t: @table,
+          b: @s3ds.bucket.name,
+          x: @prefix
+        }
         params[:q] = @statement.stripped_source.chomp(';') if @statement
         params[:f] = @format if @format
         params[:C] = nil if @gzip
@@ -285,8 +306,8 @@ module Bricolage
         Pathname(__dir__).parent.parent + "libexec/mys3dump.jar"
       end
 
-      def build_cmd(options)
-        ['java'] + options.flat_map {|k, v| v ? ["-#{k}", v.to_s] : ["-#{k}"] }
+      def build_cmd(environ, options)
+        [environ, 'java'] + options.flat_map {|k, v| v ? ["-#{k}", v.to_s] : ["-#{k}"] }
       end
     end
 

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.20.4'
+  VERSION = '5.20.5'
 end

data/test/home/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ../..
   specs:
-    bricolage (5.20.3)
+    bricolage (5.20.5)
       aws-sdk (~> 2)
       mysql2
       pg (~> 0.18.0)

data/test/home/subsys/my-import.job

@@ -3,7 +3,7 @@ src-ds: mysql
 src-tables:
     users: main.users
 s3-ds: s3
-s3-prefix: shimpeko/test-abc-
+s3-prefix: $test_schema/test-abc-
 gzip: true
 dump-options:
     src_zone_offset: "+00:00"
@@ -24,7 +24,7 @@ options:
     #acceptanydate: true
     #dateformat: "auto"
     #timeformat: "auto"
-vacuum-sort: true
+#vacuum-sort: true
 analyze: true
 grant:
     privilege: select

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.20.4
+  version: 5.20.5
 platform: ruby
 authors:
 - Minero Aoki