bricolage 5.24.3 → 5.24.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d7ca2ab62113b959d79f3102daa165cd7e843253
-  data.tar.gz: 9bac20ab47ac97242583a9075b5799a63ef07f52
+  metadata.gz: 832c3083b42f0c549ed2dbc8441716b16b093d3e
+  data.tar.gz: 50fd92b977334362676bd587bbd02fed46327213
 SHA512:
-  metadata.gz: 4250720a52ba659add07a834c4a91f5d4d87165cf51bb4caeff78a93902173c2b2c55e244d42ad5f5fadeb90404743fbc63bd9b969f30241aa45ee19e8547fc7
-  data.tar.gz: 6b52d33d3dd7f0484248f6dbf3b0edc20f8509c81c63efcfae600eabe85b5efc95d29d7f97c57d5d2d0814d6b7f5a65bdb6d61d0d9fe161a0c20ab5e3cadda7c
+  metadata.gz: 1f0ed17c1a0e745cc5b54beb0bc434a5922a2f2767607002d40ba2f3599c31258b8dfb54e5b45a2cbd530cd49302d1a0b35c22d19815e77bb96946a79af7514e
+  data.tar.gz: e3ba6c8d1020d46e96e1e110e759e4022f3d63243f9d621e88278c0067337c574dab9062df3124c667a915ce4496cb6c260d8b741b33c46e199d48c636cd33d7

data/lib/bricolage/mysqldatasource.rb

@@ -230,23 +230,16 @@ module Bricolage
       end
 
       def s3export
-        cmd = build_cmd(environment_variables, command_parameters)
-        ds.logger.info "[CMD] #{format_cmd(cmd)}"
-        Open3.popen2e(*cmd) do |input, output, thread|
-          input.close
-          output.each do |line|
-            puts line
-          end
-          unless thread.value.success?
-            raise JobFailure, "mys3dump failed (status #{thread.value.to_i})"
-          end
+        cmd = build_cmd(command_parameters)
+        ds.logger.info "[CMD] #{cmd}"
+        out, st = Open3.capture2e(environment_variables, cmd)
+        ds.logger.info "[CMDOUT] #{out}"
+        unless st.success?
+          msg = extract_exception_message(out)
+          raise JobFailure, "mys3dump failed (status: #{st.to_i}): #{msg}"
         end
       end
 
-      def format_cmd(cmd)
-        cmd[1..-1].join(' ')   # do not show env
-      end
-
       def environment_variables
         {
           'AWS_ACCESS_KEY_ID' => @s3ds.access_key,
@@ -306,8 +299,16 @@ module Bricolage
         Pathname(__dir__).parent.parent + "libexec/mys3dump.jar"
       end
 
-      def build_cmd(environ, options)
-        [environ, 'java'] + options.flat_map {|k, v| v ? ["-#{k}", v.to_s] : ["-#{k}"] }
+      def build_cmd(options)
+        (['java'] + options.flat_map {|k, v| v ? ["-#{k}", v.to_s] : ["-#{k}"] }.map {|o| %Q("#{o}") }).join(" ")
+      end
+
+      def extract_exception_message(out)
+        out.lines do |line|
+          if /^.*Exception: (?<msg>.*)$/ =~ line
+            return msg
+          end
+        end
       end
     end
 

data/lib/bricolage/sqlstatement.rb

@@ -245,7 +245,7 @@ module Bricolage
     end
 
     def parse(source)
-      parse_meta(read_mata(source))
+      parse_meta(read_meta(source))
     end
 
     private
@@ -279,7 +279,7 @@ module Bricolage
 
     KEYS_RE = /\A--(?:#{DECLARATIONS.keys.join('|')}):/
 
-    def read_mata(source)
+    def read_meta(source)
       lines = ''
       source.each_line do |line|
         case line
@@ -287,7 +287,7 @@ module Bricolage
           lines.concat line.sub(/\A--/, '')
         when /\A--([\w\-]+):/
           key = $1
-          raise ParameterError, "unknown SQL meta data: #{source.name}: #{key}"
+          raise ParameterError, "unknown SQL meta data: #{source}: #{key}"
         when /\A--/
           # skip non-metadata comments
           lines.concat "\n"

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.24.3'
+  VERSION = '5.24.4'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.24.3
+  version: 5.24.4
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-10 00:00:00.000000000 Z
+date: 2017-12-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -208,7 +208,6 @@ files:
 - test/home/data/20141002-1355_02.txt
 - test/home/data/test.txt
 - test/home/jobnet-test.rb
-- test/home/subsys/20170214_1225_0_strload.data.json.gz
 - test/home/subsys/d.ct
 - test/home/subsys/insert.sql.job
 - test/home/subsys/item_pv_acc.ct
@@ -237,9 +236,6 @@ files:
 - test/home/subsys/separated.job
 - test/home/subsys/separated.sql
 - test/home/subsys/some_view-cv.sql.job
-- test/home/subsys/streaming_load.ct
-- test/home/subsys/streaming_load.job
-- test/home/subsys/streaming_load_wk.ct
 - test/home/subsys/unified.jobnet
 - test/home/subsys/unified.sql.job
 - test/home/subsys/users.ct

data/test/home/subsys/streaming_load.ct

@@ -1,4 +0,0 @@
-create table streaming_load (
-    col1 int
-    , col2 varchar(32)
-)

data/test/home/subsys/streaming_load.job

@@ -1,21 +0,0 @@
-class: streaming_load
-
-s3-ds: s3
-keep-ctl: true
-ctl-prefix: /strload/ctl/
-queue-path: /strload/queue/
-persistent-path: /strload/year=%Y/month=%m/day=%d/
-file-name: "%Y%m%d_%H%M_%Q_%*.gz"
-
-redshift-ds: dwh_db
-dest-table: $test_schema.streaming_load
-work-table: $test_schema.streaming_load_wk
-log-table: $test_schema.streaming_load_l
-load-options:
-  json: "auto"
-  gzip: true
-  maxerror: 0
-  timeformat: "auto"
-  truncatecolumns: true
-  acceptinvchars: " "
-skip-work: true

data/test/home/subsys/streaming_load_wk.ct

@@ -1,4 +0,0 @@
-create table streaming_load_wk (
-    col1 int
-    , col2 varchar(32)
-)